diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 95f58deac..1ce0942d0 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -22,7 +22,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | | CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | NOT RECORDED | [#1333](https://github.com/lidge-jun/opencodex/pull/1333) | MERGED TO `dev` at `025c37916225dd685d9217e5b40190600f06d278`; POST-MERGE HARDENING [#1343](https://github.com/lidge-jun/opencodex/pull/1343) MERGED at `eee2dab4d1bbacefce56057adad51d734f346702`; FINAL CLOSURE GATE [#1348](https://github.com/lidge-jun/opencodex/pull/1348) | -| CL-03 | — | — | — | — | NOT STARTED; AUTHORIZATION ACTIVATES ON #1348 MERGE AFTER GREEN CI/REVIEW | +| CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | `003f7402f49bfe8dd710a7beba52f717051bfadf` | [#1352](https://github.com/lidge-jun/opencodex/pull/1352) | DRAFT PR OPEN (implementation; not accepted) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -156,4 +156,37 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-00: **ACCEPTED** (merged #1286). - CL-01: **MERGED** via #1320 at `4bb249b756abd468c675d2d92fffe4da95ad3e2a`. - CL-02: **MERGED** via #1333 at `025c37916225dd685d9217e5b40190600f06d278`; post-merge hardening #1343 is also **MERGED** at `eee2dab4d1bbacefce56057adad51d734f346702`; final closure is tracked in #1348. -- CL-03: **NOT STARTED**. Authorization is activated by merge of #1348 after green required CI and zero unresolved valid CodeRabbit findings. +- CL-03: **DRAFT PR OPEN** ([#1352](https://github.com/lidge-jun/opencodex/pull/1352)) on + `feat/cl-03-live-route-probes` from `4f746d13799888ea0a8c7a111aa2ad61c2126ea0`; + implementation head `003f7402f49bfe8dd710a7beba52f717051bfadf`. Not accepted. +- CL-04: **NOT STARTED** (blocked until CL-03 independent acceptance and review reconciliation). + +## CL-03 implementation log (2026-08-09) + +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting/base SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (#1348 merge on `dev`) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) +- **Scope:** bounded live-route probes for `live_route_compatibility`; live manifest + authority; `RouteSubjectV1` builder; `LabDestinationV1` / credential lease sandbox; + inert tool/MCP stubs; live runner/executor; `observe/from-live` persistence; + projection applicability for route subjects. +- **Explicitly out of scope:** CL-04 CLI/API, CL-05 UI, CL-06 profile fields, Fabric, + shadow/automatic probing, production request-path probes. + +### CL-03 validation (local, 2026-08-09) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: 17/17 passed +- `bun test tests/lab-evidence-ledger.test.ts`: 37/41 passed (4 Windows SQLite `EPERM` + flakes in `wipeSqlite`; `rebuild.ts` unchanged vs `upstream/dev` — pre-existing) +- `bun test tests/lab-live-probe.test.ts`: 19/19 passed +- `bun test tests/lab-live-sandbox.test.ts`: 17/17 passed +- `bun run privacy:scan`: passed +- Cross-platform CI on #1352: pending at open time + +### CL-03 blockers + +- Independent acceptance review not performed +- Draft PR review findings not yet reconciled +- Full local ledger suite not green on Windows host (pre-existing SQLite EPERM) diff --git a/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md b/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md new file mode 100644 index 000000000..0e599345e --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/023_live_v1_manifest_authority.md @@ -0,0 +1,56 @@ +# CL-03 live V1 manifest authority + +This document closes the executable semantics for the initial +`live_route_compatibility` scenarios. It is normative for CL-03. + +The machine-readable source of truth is +[`024_live_v1_cases.json`](./024_live_v1_cases.json). It contains 10 frozen +live-route scenarios with Lab-authored synthetic fixtures, literal expected +values, row-specific requirements, execution limits from the security contract, +artifact policy, failure rules, and domain-separated fixture digests. + +## 1. Manifest expansion + +For each entry in `cases`, CL-03 constructs `CompatibilityScenarioV1` using the +same `fixtureRef` contract as protocol V1, with authority +`024_live_v1_cases.json`. Defaults include: + +- `evidenceLayer`: `live_route_compatibility` +- `executionMode`: `live` +- `freshness.maxAgeMs`: `604800000` (7 days) +- `failureRuleSet`: `live-v1-default` + +Environmental blockers (`authentication_blocked`, `quota_blocked`, +`network_failure`, `provider_transient`, `region_blocked`, `timeout`, +`budget_exhausted`) map to `verdictEffect: none` and never produce compatibility +degradation. + +## 2. Live suites + +| Suite | Live scenario | +|---|---| +| `responses-core` | `responses-core.live.basic-turn` | +| `chat-core` | `chat-core.live.basic-turn` | +| `anthropic-core` | `anthropic-core.live.basic-turn` | +| `tools-core` | `tools-core.live.function-round-trip`, `tools-core.live.custom-freeform-round-trip` | +| `codex-core` | `codex-core.live.tool-turn`, `codex-core.live.custom-tool-turn` | +| `vision-core` | `vision-core.live.synthetic-ocr` | +| `reasoning-core` | `reasoning-core.live.replay` | +| `mcp-core` | `mcp-core.live.synthetic-tool` | + +## 3. Route subject and sandbox + +Live evidence uses `RouteSubjectV1` with opaque `endpointFingerprint` and +`providerInstanceFingerprint` derived via `localFingerprint()` from immutable +destination snapshots. Raw URLs, credentials, and DNS results are never +persisted. + +The live sandbox rejects proxy environment variables, permits only +`TZ=UTC` and `NO_COLOR=1`, enforces security-contract resource ceilings, and +uses injectable transport in tests. + +## 4. CL-03 boundary + +CL-03 implements the live runner, sandbox, route subject builder, persistence +seam, and projection applicability for `live_route_compatibility`. It does not +implement CL-04 CLI/API. diff --git a/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json new file mode 100644 index 000000000..4301deeca --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/024_live_v1_cases.json @@ -0,0 +1,175 @@ +{ + "schemaVersion": 1, + "authority": "CL-03 live manifest authority", + "sourceCommit": "4f746d13799888ea0a8c7a111aa2ad61c2126ea0", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "live-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "auth-blocked", "match": ["auth_blocked", "authentication_blocked"], "classification": "authentication_blocked", "secondaryCode": "auth_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "quota-blocked", "match": ["quota_blocked"], "classification": "quota_blocked", "secondaryCode": "quota_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "network-blocked", "match": ["network_blocked", "network_failure"], "classification": "network_failure", "secondaryCode": "network_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "region-blocked", "match": ["region_blocked"], "classification": "region_blocked", "secondaryCode": "region_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "transient-blocked", "match": ["provider_transient"], "classification": "provider_transient", "secondaryCode": "provider_transient", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "live_route_compatibility", + "verificationRole": "required", + "executionMode": "live", + "freshness": { "maxAgeMs": 604800000 }, + "executionLimits": { + "totalTimeoutMs": 120000, + "connectTimeoutMs": 10000, + "firstByteTimeoutMs": 30000, + "inactivityTimeoutMs": 30000, + "maxRequests": 16, + "maxInputBytes": 8388608, + "maxOutputBytes": 16777216, + "maxOutputTokens": 32768, + "maxToolCalls": 32, + "maxMemoryBytes": 536870912, + "maxChildProcesses": 0, + "maxArtifacts": 16, + "perArtifactBytes": 262144, + "aggregateArtifactBytes": 1048576 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_live_v1" + }, + "failureRuleSet": "live-v1-default" + }, + "cases": [ + { + "id": "responses-core.live.basic-turn", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "rsp-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "rsp-live-basic-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]}", "digest": "8008bc78564ccf7676fba34035df94bf3f342f186ad7778f75efa71058604429" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.live.basic-turn", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "chat-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.live.basic-turn", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "anthropic-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "tools-core.live.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-fn-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"function\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"tool_choice\":{\"type\":\"function\",\"name\":\"lookup\"}}", "digest": "50b83cb763832dee3c68def08fd478c40eaf52ee2f653392e9a42935eea4a682" }, + "fixture": { "id": "tools-live-fn-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"function_call\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}]}", "digest": "18f02e77d01a179de9275a5a8417c202a0d87ec39717af0bd400f3d111cb8fa3" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true } + ] + }, + { + "id": "tools-core.live.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\",\"description\":\"patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "005ec44b618015bc42cf465ac91df5028bfe591cd0b9f0c99dfc408883584cc5" }, + "fixture": { "id": "tools-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\"}]}", "digest": "037eee2f48fad832ddfa59e9ea679878d034ad1bbb087c7511d969253e5f1f49" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "codex-core.live.tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-tool-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-live-tool-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "ebfe070989d76b0cacbaf16379ac9fb0d51d5103a2778e994c03495498484386" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "codex-core.live.custom-tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "c30efedea137e24e0e259ac5dd8a44ed5d329454e37e68a00b6de280ec61682c" }, + "fixture": { "id": "codex-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"PATCH\"}]}", "digest": "b5fc9cc273b8581f62f17b83d51e5b565f379ef3950865e687b91d9b3ee946a6" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"PATCH","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "vision-core.live.synthetic-ocr", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, + "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, + "assertions": [ + { "id": "nonce", "operator": "json_path_equals", "selector": "/client/response/json/output/0/content/0/text", "expected": "{\"nonce\":\"OCR-NONCE-42\"}", "required": true } + ] + }, + { + "id": "reasoning-core.live.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector", "reasoning_replay", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "reasoning-live-replay-vector", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true }, + { "id": "private-absent", "operator": "json_path_absent", "selector": "/client/response/json/reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.live.synthetic-tool", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "mcp-live-stub", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"ECHO\"}],\"isError\":false}}", "digest": "dd8285346f7f93b9529e2fbaabb8ffc420ca2ce0c04ca19b656ed286268fcbd4" }, + "assertions": [ + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"ECHO"}],"isError":false}, "required": true } + ] + } + ] +} \ No newline at end of file diff --git a/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md new file mode 100644 index 000000000..856abbbce --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/054_cl03_implementation_record.md @@ -0,0 +1,63 @@ +# CL-03 implementation record + +## Branch and revisions + +- **Branch:** `feat/cl-03-live-route-probes` +- **Starting SHA:** `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` (upstream `dev` / #1348 merge) +- **Implementation head:** `003f7402f49bfe8dd710a7beba52f717051bfadf` +- **PR:** [#1352](https://github.com/lidge-jun/opencodex/pull/1352) (DRAFT → `lidge-jun/opencodex:dev`) + +## Scope delivered + +1. Live manifest authority (`023_live_v1_manifest_authority.md`, `024_live_v1_cases.json`, runtime copy) +2. Route subject builder (`src/lab/subject/`) +3. Live sandbox (`src/lab/live/` — destination, credential lease, transport, executor, runner) +4. Persistence seam (`src/lab/observe/from-live.ts`) +5. Projection applicability (`routeSubjectApplicableToRequirements` in `verification.ts`) +6. Tests (`tests/lab-live-sandbox.test.ts`, `tests/lab-live-probe.test.ts`) + +## Security design summary + +- **Network:** immutable in-memory `LabDestinationV1` snapshot; DNS/policy before credentials; + connect only to approved address set; redirects rejected; metadata/link-local forbidden; + private/loopback requires route `allowPrivateNetwork` plus explicit `labRunApproval`; + inherited proxy env vars rejected. +- **Credentials:** opaque `LabCredentialLeaseV1` bound to destination + transport + budget; + no secret bytes in Lab code; auth failure → `authentication_blocked` / `BLOCKED`. +- **Tools/MCP:** inert Lab-authored tools; loopback MCP stub only; no user `mcpServers`. +- **Process:** sandbox env `TZ=UTC`, `NO_COLOR=1` only; no child processes; resource ceilings enforced. + +## Frozen live scenarios (10) + +- `responses-core.live.basic-turn` +- `chat-core.live.basic-turn` +- `anthropic-core.live.basic-turn` +- `tools-core.live.function-round-trip` +- `tools-core.live.custom-freeform-round-trip` +- `codex-core.live.tool-turn` +- `codex-core.live.custom-tool-turn` +- `vision-core.live.synthetic-ocr` +- `reasoning-core.live.replay` +- `mcp-core.live.synthetic-tool` + +## Validation (2026-08-09) + +| Check | Result | +|---|---| +| `bun x tsc --noEmit` | pass | +| `tests/lab-conformance-harness.test.ts` | 17/17 | +| `tests/lab-evidence-ledger.test.ts` | 37/41 (4 Windows SQLite EPERM flakes; pre-existing) | +| `tests/lab-live-probe.test.ts` | 19/19 | +| `tests/lab-live-sandbox.test.ts` | 17/17 | +| `bun run privacy:scan` | pass | + +## Explicitly not started + +- CL-04 Lab CLI or management/read APIs +- CL-05 Compatibility Matrix UI +- CL-06 Routing Profile compatibility policy + +## Acceptance status + +Implementation only — **not accepted**. CL-04 remains blocked until independent CL-03 +acceptance and review reconciliation. diff --git a/scripts/gen-live-digests.ts b/scripts/gen-live-digests.ts new file mode 100644 index 000000000..eeda8dcab --- /dev/null +++ b/scripts/gen-live-digests.ts @@ -0,0 +1,52 @@ +import { fixtureDigest } from "../src/lab/conformance/digest"; + +const fixtures: Record = { + "rsp-live-basic-request": + '{"model":"fixture-model","input":"PING","stream":false}', + "rsp-live-basic-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"OK"}]}]}', + "chat-live-basic-request": + '{"model":"fixture-model","input":"PING","stream":true}', + "chat-live-basic-upstream": + 'data: {"choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', + "anthropic-live-basic-request": + '{"model":"fixture-model","messages":[{"role":"user","content":"PING"}],"max_tokens":32,"stream":true}', + "anthropic-live-basic-upstream": + 'data:{"type":"response.output_text.delta","delta":"OK"}\n\ndata:{"type":"response.completed","response":{"id":"resp_fixture","status":"completed","usage":{"input_tokens":1,"output_tokens":1}}}\n\n', + "tools-live-fn-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"function","name":"lookup","parameters":{"type":"object","properties":{"q":{"type":"string"}},"required":["q"]}}],"tool_choice":{"type":"function","name":"lookup"}}', + "tools-live-fn-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"function_call","call_id":"call_fixture","name":"lookup","arguments":"{\\"q\\":\\"x\\"}"}]}', + "tools-live-fn-result-request": + '{"model":"fixture-model","input":[{"type":"function_call_output","call_id":"call_fixture","output":"RESULT"}],"stream":false}', + "tools-live-fn-result-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"DONE"}]}]}', + "tools-live-custom-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"custom","name":"apply_patch","description":"patch"}],"tool_choice":{"type":"custom","name":"apply_patch"}}', + "tools-live-custom-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"custom_tool_call","call_id":"call_fixture","name":"apply_patch","input":"*** Begin Patch\\n*** End Patch"}]}', + "tools-live-custom-result-request": + '{"model":"fixture-model","input":[{"type":"custom_tool_call_output","call_id":"call_fixture","output":"PATCHED"}],"stream":false}', + "tools-live-custom-result-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"DONE"}]}]}', + "codex-live-tool-request": + '{"model":"fixture-model","input":"PING","stream":true}', + "codex-live-tool-upstream": + 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_fixture","function":{"name":"lookup","arguments":"{\\"q\\":\\"x\\"}"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n', + "codex-live-custom-request": + '{"model":"fixture-model","input":"PING","stream":false,"tools":[{"type":"custom","name":"apply_patch"}],"tool_choice":{"type":"custom","name":"apply_patch"}}', + "codex-live-custom-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"call_fixture","type":"custom_tool_call","call_id":"call_fixture","name":"apply_patch","input":"PATCH"}]}', + "vision-live-ocr-request": + '{"model":"fixture-model","input":[{"role":"user","content":[{"type":"input_text","text":"OCR-NONCE-42"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgo=","detail":"high"}]}],"stream":false}', + "vision-live-ocr-upstream": + '{"id":"resp_fixture","status":"completed","output":[{"id":"msg_fixture","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"{\\"nonce\\":\\"OCR-NONCE-42\\"}"}]}]}', + "reasoning-live-replay-vector": + '{"turn1":{"reasoning":{"id":"rs_fixture","text":"PLAN","signature":"sig_fixture"},"toolCall":{"callId":"call_fixture"}},"turn2":{"toolResult":{"callId":"call_fixture","output":"RESULT"}}}', + "mcp-live-stub": + '{"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"},"result":{"content":[{"type":"text","text":"ECHO"}],"isError":false}}', +}; + +for (const [id, bytes] of Object.entries(fixtures)) { + console.log(id, fixtureDigest(new TextEncoder().encode(bytes))); +} diff --git a/src/lab/conformance/fixtures/live-v1-cases.json b/src/lab/conformance/fixtures/live-v1-cases.json new file mode 100644 index 000000000..4301deeca --- /dev/null +++ b/src/lab/conformance/fixtures/live-v1-cases.json @@ -0,0 +1,175 @@ +{ + "schemaVersion": 1, + "authority": "CL-03 live manifest authority", + "sourceCommit": "4f746d13799888ea0a8c7a111aa2ad61c2126ea0", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "live-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "auth-blocked", "match": ["auth_blocked", "authentication_blocked"], "classification": "authentication_blocked", "secondaryCode": "auth_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "quota-blocked", "match": ["quota_blocked"], "classification": "quota_blocked", "secondaryCode": "quota_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "network-blocked", "match": ["network_blocked", "network_failure"], "classification": "network_failure", "secondaryCode": "network_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "region-blocked", "match": ["region_blocked"], "classification": "region_blocked", "secondaryCode": "region_blocked", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "transient-blocked", "match": ["provider_transient"], "classification": "provider_transient", "secondaryCode": "provider_transient", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "live_route_compatibility", + "verificationRole": "required", + "executionMode": "live", + "freshness": { "maxAgeMs": 604800000 }, + "executionLimits": { + "totalTimeoutMs": 120000, + "connectTimeoutMs": 10000, + "firstByteTimeoutMs": 30000, + "inactivityTimeoutMs": 30000, + "maxRequests": 16, + "maxInputBytes": 8388608, + "maxOutputBytes": 16777216, + "maxOutputTokens": 32768, + "maxToolCalls": 32, + "maxMemoryBytes": 536870912, + "maxChildProcesses": 0, + "maxArtifacts": 16, + "perArtifactBytes": 262144, + "aggregateArtifactBytes": 1048576 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_live_v1" + }, + "failureRuleSet": "live-v1-default" + }, + "cases": [ + { + "id": "responses-core.live.basic-turn", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "rsp-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "rsp-live-basic-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]}", "digest": "8008bc78564ccf7676fba34035df94bf3f342f186ad7778f75efa71058604429" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.live.basic-turn", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "chat-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.live.basic-turn", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "anthropic-live-basic-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-live-basic-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "tools-core.live.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-fn-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"function\",\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"tool_choice\":{\"type\":\"function\",\"name\":\"lookup\"}}", "digest": "50b83cb763832dee3c68def08fd478c40eaf52ee2f653392e9a42935eea4a682" }, + "fixture": { "id": "tools-live-fn-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"function_call\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}]}", "digest": "18f02e77d01a179de9275a5a8417c202a0d87ec39717af0bd400f3d111cb8fa3" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true } + ] + }, + { + "id": "tools-core.live.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "tools-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\",\"description\":\"patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "005ec44b618015bc42cf465ac91df5028bfe591cd0b9f0c99dfc408883584cc5" }, + "fixture": { "id": "tools-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\"}]}", "digest": "037eee2f48fad832ddfa59e9ea679878d034ad1bbb087c7511d969253e5f1f49" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "codex-core.live.tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-tool-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-live-tool-upstream", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_fixture\",\"function\":{\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "ebfe070989d76b0cacbaf16379ac9fb0d51d5103a2778e994c03495498484386" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "codex-core.live.custom-tool-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["inert_tools", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "codex-live-custom-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false,\"tools\":[{\"type\":\"custom\",\"name\":\"apply_patch\"}],\"tool_choice\":{\"type\":\"custom\",\"name\":\"apply_patch\"}}", "digest": "c30efedea137e24e0e259ac5dd8a44ed5d329454e37e68a00b6de280ec61682c" }, + "fixture": { "id": "codex-live-custom-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"call_fixture\",\"type\":\"custom_tool_call\",\"call_id\":\"call_fixture\",\"name\":\"apply_patch\",\"input\":\"PATCH\"}]}", "digest": "b5fc9cc273b8581f62f17b83d51e5b565f379ef3950865e687b91d9b3ee946a6" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"apply_patch","arguments":"PATCH","kind":"custom","ordinal":0}, "required": true } + ] + }, + { + "id": "vision-core.live.synthetic-ocr", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["synthetic_image", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "initiatingRequest": { "id": "vision-live-ocr-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"OCR-NONCE-42\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}],\"stream\":false}", "digest": "8e498fc17b41928a751f65157a556da17e4cc4f2e728d15db498eac81b59fdc3" }, + "fixture": { "id": "vision-live-ocr-upstream", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"{\\\"nonce\\\":\\\"OCR-NONCE-42\\\"}\"}]}]}", "digest": "297d031303467c16f23635fc7c3fc5ebd7075e157a51e0623b975e11ce6ffd05" }, + "assertions": [ + { "id": "nonce", "operator": "json_path_equals", "selector": "/client/response/json/output/0/content/0/text", "expected": "{\"nonce\":\"OCR-NONCE-42\"}", "required": true } + ] + }, + { + "id": "reasoning-core.live.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector", "reasoning_replay", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "reasoning-live-replay-vector", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true }, + { "id": "private-absent", "operator": "json_path_absent", "selector": "/client/response/json/reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.live.synthetic-tool", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub", "live_transport"], "platforms": [], "routePreconditions": ["lab_run_approval"] }, + "fixture": { "id": "mcp-live-stub", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"ECHO\"}],\"isError\":false}}", "digest": "dd8285346f7f93b9529e2fbaabb8ffc420ca2ce0c04ca19b656ed286268fcbd4" }, + "assertions": [ + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"ECHO"}],"isError":false}, "required": true } + ] + } + ] +} \ No newline at end of file diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 6bdb44fbd..30cc0c963 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -13,6 +13,11 @@ export type FailureClassification = | "protocol_failure" | "capability_failure" | "behavioral_failure" + | "authentication_blocked" + | "quota_blocked" + | "region_blocked" + | "network_failure" + | "provider_transient" | "inconclusive"; export interface FixtureRecord { @@ -33,7 +38,7 @@ export interface AssertionSpec { export interface ExpectedFailureSpec { controlKind: "conformance_negative_control" | "capability_absence_control"; - expectedClass: string; + expectedClass: FailureClassification; expectedCode: string; assertionIds: string[]; onMatch: "pass" | "unsupported"; @@ -161,6 +166,18 @@ export interface ScenarioRunResult { executionContext: ProtocolExecutionContextV1; } +/** All eight live-route compatibility suites frozen by CL-03 Live V1. */ +export const CL03_LIVE_SUITES = [ + "responses-core", + "chat-core", + "anthropic-core", + "tools-core", + "codex-core", + "vision-core", + "reasoning-core", + "mcp-core", +] as const; + /** All eight protocol-conformance suites frozen by CL-00 Protocol V1. */ export const CL01_SUITES = [ "responses-core", diff --git a/src/lab/index.ts b/src/lab/index.ts index 2515f8481..bbce80e51 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -17,3 +17,19 @@ export * from "./conformance/suite-manifest"; export * from "./ledger/artifact-refs"; export * from "./projection/verification"; export * from "./observe/from-conformance"; +export * from "./observe/from-live"; +export * from "./live/manifest"; +export * from "./live/runner"; +export * from "./live/executor"; +export * from "./live/types"; +export * from "./live/destination"; +export * from "./live/sandbox"; +export * from "./live/transport"; +export * from "./live/credential-lease"; +export * from "./live/inert-tools"; +export * from "./live/mcp-loopback"; +export * from "./live/suite-manifest"; +export * from "./subject/route-subject"; +export * from "./subject/behavior-fingerprint"; +export * from "./subject/installation-salt"; +export { CL03_LIVE_SUITES } from "./conformance/types"; diff --git a/src/lab/live/credential-lease.ts b/src/lab/live/credential-lease.ts new file mode 100644 index 000000000..1e557777d --- /dev/null +++ b/src/lab/live/credential-lease.ts @@ -0,0 +1,53 @@ +import { LAB_CREDENTIAL_LEASE, type LabCredentialLeaseV1, type LabDestinationV1 } from "./types"; + +export class LabCredentialError extends Error { + override readonly name = "LabCredentialError"; + constructor(message: string, readonly code: "harness_failure" | "budget_exhausted") { super(message); } +} + +interface LeaseState { destinationFingerprint: string; transportId: string; remaining: number } +const LEASE_STATE = new WeakMap(); +type InternalLease = LabCredentialLeaseV1 & { toJSON(): never }; + +export interface CreateLeaseOptions { + destination: LabDestinationV1; + transportId?: string; + budget?: number; +} + +/** Trusted credential owner may associate secret state with this opaque capability outside Lab. */ +export function createCredentialLease(opts: CreateLeaseOptions): LabCredentialLeaseV1 { + const budget = opts.budget ?? 1; + if (!Number.isInteger(budget) || budget <= 0) throw new LabCredentialError("invalid credential lease budget", "harness_failure"); + let lease!: InternalLease; + lease = { + [LAB_CREDENTIAL_LEASE]: true, + get remainingRequests(): number { return LEASE_STATE.get(lease)?.remaining ?? 0; }, + consume(): void { + const state = LEASE_STATE.get(lease); + if (!state) throw new LabCredentialError("unknown credential lease", "harness_failure"); + if (state.remaining <= 0) throw new LabCredentialError("credential lease exhausted", "budget_exhausted"); + state.remaining -= 1; + }, + toJSON(): never { throw new LabCredentialError("credential lease is non-serializable", "harness_failure"); }, + }; + LEASE_STATE.set(lease, { + destinationFingerprint: opts.destination.fingerprint, + transportId: opts.transportId ?? "default", + remaining: budget, + }); + return Object.freeze(lease); +} + +export function isCredentialLease(value: unknown): value is LabCredentialLeaseV1 { + return typeof value === "object" && value !== null && LEASE_STATE.has(value as object) + && (value as LabCredentialLeaseV1)[LAB_CREDENTIAL_LEASE] === true; +} + +export function assertLeaseScope(lease: LabCredentialLeaseV1, destination: LabDestinationV1, transportId = "default"): void { + const state = LEASE_STATE.get(lease as object); + if (!state) throw new LabCredentialError("unknown credential lease", "harness_failure"); + if (state.destinationFingerprint !== destination.fingerprint || state.transportId !== transportId) { + throw new LabCredentialError("credential lease scope mismatch", "harness_failure"); + } +} diff --git a/src/lab/live/destination.ts b/src/lab/live/destination.ts new file mode 100644 index 000000000..56d9bc444 --- /dev/null +++ b/src/lab/live/destination.ts @@ -0,0 +1,155 @@ +import { isIP } from "node:net"; +import { assessUrlDestination, resolvePublicAddresses } from "../../lib/destination-policy"; +import { localFingerprint } from "../digest"; +import { readInstallationSalt } from "../subject/installation-salt"; +import { TransportError } from "./transport"; +import type { DnsResolver, LabDestinationV1 } from "./types"; + +export class LabDestinationError extends Error { + override readonly name = "LabDestinationError"; + constructor(message: string, readonly code: string) { super(message); } +} + +function normalizeBasePath(pathname: string): string { + if (!pathname || pathname === "/") return ""; + const trimmed = pathname.endsWith("/") ? pathname.slice(0, -1) : pathname; + return trimmed.startsWith("/") ? trimmed : `/${trimmed}`; +} + +function addressUrl(scheme: "http" | "https", address: string): string { + return `${scheme}://${isIP(address) === 6 ? `[${address}]` : address}/`; +} + +function validateResolvedAddress( + scheme: "http" | "https", + address: { address: string; family: 4 | 6 }, + allowPrivateNetwork: boolean, + labRunApproval: boolean, +): boolean { + const actualFamily = isIP(address.address); + if ((actualFamily !== 4 && actualFamily !== 6) || actualFamily !== address.family) { + throw new LabDestinationError("resolver returned an invalid address/family", "harness_failure"); + } + const assessment = assessUrlDestination(addressUrl(scheme, address.address)); + if (!assessment) throw new LabDestinationError("resolver address could not be classified", "harness_failure"); + if (["metadata", "link-local", "unspecified"].includes(assessment.kind)) { + throw new LabDestinationError("blocked resolved destination", "harness_failure"); + } + const privateAnswer = ["private", "loopback", "localhost"].includes(assessment.kind); + if (privateAnswer && (!allowPrivateNetwork || !labRunApproval)) { + throw new LabDestinationError("resolved private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); + } + return privateAnswer; +} + +function canonicalAddresses(addresses: ReadonlyArray<{ address: string; family: 4 | 6 }>): Array<{ address: string; family: 4 | 6 }> { + const unique = new Map(); + for (const row of addresses) unique.set(`${row.family}:${row.address}`, { ...row }); + return [...unique.values()].sort((a, b) => a.family - b.family || (a.address < b.address ? -1 : a.address > b.address ? 1 : 0)); +} + +function addressSetKey(addresses: ReadonlyArray<{ address: string; family: 4 | 6 }>): string { + return canonicalAddresses(addresses).map((row) => `${row.family}:${row.address}`).join(","); +} + +async function withResolutionTimeout(timeoutMs: number, operation: () => Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + operation(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new TransportError("connect_timeout", "destination resolution timed out")), timeoutMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +export interface CreateDestinationOptions { + baseUrl: string; + allowPrivateNetwork?: boolean; + labRunApproval?: boolean; + resolve?: DnsResolver; + connectTimeoutMs?: number; + configDir?: string; +} + +/** Resolve once, policy-check every answer, then freeze the exact snapshot used by all later stages. */ +export async function createLabDestination(opts: CreateDestinationOptions): Promise { + let parsed: URL; + try { parsed = new URL(opts.baseUrl); } catch { throw new LabDestinationError("invalid provider baseUrl", "harness_failure"); } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new LabDestinationError("unsupported URL scheme", "harness_failure"); + if (parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new LabDestinationError("provider destination may not contain userinfo, query, or fragment", "harness_failure"); + } + const scheme = parsed.protocol === "https:" ? "https" as const : "http" as const; + const literal = assessUrlDestination(parsed.toString()); + if (literal && ["metadata", "link-local", "unspecified"].includes(literal.kind)) { + throw new LabDestinationError("blocked destination", "harness_failure"); + } + const allowPrivate = opts.allowPrivateNetwork === true; + const approved = opts.labRunApproval === true; + const literalPrivate = Boolean(literal && ["private", "loopback", "localhost"].includes(literal.kind)); + if (literalPrivate && (!allowPrivate || !approved)) { + throw new LabDestinationError("private network requires allowPrivateNetwork and labRunApproval", "harness_failure"); + } + + const connectTimeoutMs = opts.connectTimeoutMs ?? 10_000; + if (!Number.isInteger(connectTimeoutMs) || connectTimeoutMs <= 0) { + throw new LabDestinationError("invalid connect timeout", "harness_failure"); + } + + let addresses: Array<{ address: string; family: 4 | 6 }>; + let privateNetwork = literalPrivate; + if (opts.resolve) { + try { addresses = await withResolutionTimeout(connectTimeoutMs, () => opts.resolve!(parsed.hostname)); } + catch (error) { + if (error instanceof TransportError || error instanceof LabDestinationError) throw error; + throw new LabDestinationError("DNS resolution failed", "network_blocked"); + } + if (addresses.length === 0) throw new LabDestinationError("DNS resolution returned no addresses", "network_blocked"); + for (const row of addresses) privateNetwork = validateResolvedAddress(scheme, row, allowPrivate, approved) || privateNetwork; + } else { + try { + const resolved = await withResolutionTimeout(connectTimeoutMs, () => resolvePublicAddresses(parsed.toString(), { + context: "Lab provider destination", + allowPrivateNetwork: allowPrivate && approved, + })); + addresses = resolved.addresses.map((row) => ({ address: row.address, family: row.family === 6 ? 6 as const : 4 as const })); + privateNetwork = resolved.privateNetwork || privateNetwork; + } catch (error) { + if (error instanceof TransportError || error instanceof LabDestinationError) throw error; + throw new LabDestinationError("DNS destination policy failed", "network_blocked"); + } + } + const frozenAddresses = Object.freeze(canonicalAddresses(addresses).map((row) => Object.freeze(row))); + const port = parsed.port ? Number(parsed.port) : scheme === "https" ? 443 : 80; + if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new LabDestinationError("invalid destination port", "harness_failure"); + const snapshot = { + scheme, + host: parsed.hostname.toLowerCase(), + port, + basePath: normalizeBasePath(parsed.pathname), + sniHost: parsed.hostname.toLowerCase(), + addresses: frozenAddresses, + privateNetwork, + }; + const salt = readInstallationSalt(opts.configDir); + const fingerprint = localFingerprint("endpoint", { + scheme: snapshot.scheme, host: snapshot.host, port: snapshot.port, basePath: snapshot.basePath, + }, salt); + return Object.freeze({ ...snapshot, fingerprint }); +} + +/** Detect any attempted replacement/re-resolution of the approved immutable address set. */ +export function assertDestinationAddressSet(destination: LabDestinationV1, addresses: Array<{ address: string; family: 4 | 6 }>): void { + if (addressSetKey(destination.addresses) !== addressSetKey(addresses)) { + throw new LabDestinationError("destination address set mismatch", "destination_mismatch"); + } +} + +export function assertHostSniMatch(destination: LabDestinationV1, host: string, sni?: string): void { + if (host.toLowerCase() !== destination.host) throw new LabDestinationError("host mismatch", "host_sni_mismatch"); + if (sni !== undefined && sni.toLowerCase() !== destination.sniHost) throw new LabDestinationError("SNI mismatch", "host_sni_mismatch"); +} diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts new file mode 100644 index 000000000..fe71947a9 --- /dev/null +++ b/src/lab/live/executor.ts @@ -0,0 +1,311 @@ +import { evaluateAssertions } from "../conformance/assertion"; +import { emptyObservation, finalizeObservation, setClientResponse } from "../conformance/observation"; +import { normalizeSseBytes } from "../conformance/sse-normalize"; +import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule, NormalizedEvent, NormalizedObservation } from "../conformance/types"; +import { domainHash, jcsStringify, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; +import type { RouteSubjectV1 } from "../events/types"; +import { isTrustedLabRouteExecutor } from "../../lib/lab-live-execution-authority"; +import { buildRouteSubjectV1 } from "../subject/route-subject"; +import { createLabDestination, LabDestinationError } from "./destination"; +import { expandLiveScenario, loadLiveCaseAuthority } from "./manifest"; +import { createSandboxResourceState, enforceSandboxLimits, LabSandboxError, prepareLiveSandbox } from "./sandbox"; +import { liveSuiteManifestObjectForCase } from "./suite-manifest"; +import { classifyTransportError, TransportError } from "./transport"; +import type { DnsResolver, LabRouteContext, LabTransport, LiveExecutionAuthority, LiveRunConfig, LiveScenarioRunResult, TrustedLabRouteExecutor } from "./types"; + +export interface LiveExecutorOptions { + /** Host-issued exact-route capability for evidence-eligible production execution. */ + routeExecutor?: TrustedLabRouteExecutor; + /** Test-only byte transport. Results are deliberately not evidence-eligible. */ + transport?: LabTransport; + resolve?: DnsResolver; + configDir?: string; + env?: NodeJS.ProcessEnv; +} + +interface TrustedLiveResultReceipt { + authorityDigest: string; + scenarioId: string; + suiteId: string; + scenarioManifestDigest: string; + suiteManifestDigest: string; + routeSubjectId: string; + resultDigest: string; + retryPolicy: FailureRule["retry"] | null; +} + +const TRUSTED_RESULT_RECEIPTS = new WeakMap(); +const LIVE_AUTHORITY_DOMAIN = "ocx-lab:live-authority:v1"; +const TRUSTED_LIVE_RESULT_DOMAIN = "ocx-lab:trusted-live-result:v1"; + +function trustedResultDigest(result: LiveScenarioRunResult): string { + const payload = { + scenarioId: result.scenarioId, + suite: result.suite, + startedAt: result.startedAt, + completedAt: result.completedAt, + passed: result.passed, + classification: result.classification, + secondaryCode: result.secondaryCode ?? null, + assertionResults: result.assertionResults.map((row) => ({ + id: row.id, + operator: row.operator, + required: row.required, + passed: row.passed, + observedSummary: row.observedSummary, + reason: row.reason ?? null, + })), + diagnostics: [...result.diagnostics], + routeSubject: result.routeSubject ?? null, + transportError: result.transportError ?? null, + executionAuthority: result.executionAuthority, + }; + return domainHash(TRUSTED_LIVE_RESULT_DOMAIN, jcsStringify(payload)); +} + +function receiptFor(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, retryPolicy: FailureRule["retry"] | null): TrustedLiveResultReceipt { + if (!result.routeSubject) throw new Error("trusted live result has no route subject"); + return { + authorityDigest: domainHash(LIVE_AUTHORITY_DOMAIN, jcsStringify(authority)), + scenarioId: caseRecord.id, + suiteId: caseRecord.suite, + scenarioManifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), + suiteManifestDigest: suiteManifestDigest(liveSuiteManifestObjectForCase(caseRecord, authority)), + routeSubjectId: subjectIdForSubject(result.routeSubject), + resultDigest: trustedResultDigest(result), + retryPolicy, + }; +} + +function sealTrustedLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, retryPolicy: FailureRule["retry"] | null): void { + TRUSTED_RESULT_RECEIPTS.set(result, receiptFor(result, caseRecord, authority, retryPolicy)); +} + +/** Verify the module-private execution receipt before any live result enters persistence. */ +export function assertTrustedLiveResultReceipt(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): void { + const actual = TRUSTED_RESULT_RECEIPTS.get(result); + if (!actual || result.executionAuthority !== "trusted_route") { + throw new Error("live result lacks trusted execution receipt"); + } + const expected = receiptFor(result, caseRecord, authority, actual.retryPolicy); + for (const key of Object.keys(expected) as Array) { + if (actual[key] !== expected[key]) throw new Error("live result trusted execution receipt mismatch"); + } + if (result.scenarioId !== caseRecord.id || result.suite !== caseRecord.suite) { + throw new Error("live result scenario identity mismatch"); + } +} + +/** Return retryability sealed by the executor after validating the trusted receipt. */ +export function trustedLiveResultRetryable(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority): boolean { + assertTrustedLiveResultReceipt(result, caseRecord, authority); + return TRUSTED_RESULT_RECEIPTS.get(result)?.retryPolicy === "bounded"; +} + +function liveLimitsFromAuthority(authorityLimits: Record): LiveRunConfig { + return { + totalTimeoutMs: authorityLimits.totalTimeoutMs ?? 120_000, connectTimeoutMs: authorityLimits.connectTimeoutMs ?? 10_000, + firstByteTimeoutMs: authorityLimits.firstByteTimeoutMs ?? 30_000, inactivityTimeoutMs: authorityLimits.inactivityTimeoutMs ?? 30_000, + maxRequests: authorityLimits.maxRequests ?? 16, maxInputBytes: authorityLimits.maxInputBytes ?? 8 * 1024 * 1024, + maxOutputBytes: authorityLimits.maxOutputBytes ?? 16 * 1024 * 1024, maxOutputTokens: authorityLimits.maxOutputTokens ?? 32_768, + maxToolCalls: authorityLimits.maxToolCalls ?? 32, maxMemoryBytes: authorityLimits.maxMemoryBytes ?? 512 * 1024 * 1024, + maxChildProcesses: authorityLimits.maxChildProcesses ?? 0, maxArtifacts: authorityLimits.maxArtifacts ?? 16, + perArtifactBytes: authorityLimits.perArtifactBytes ?? 256 * 1024, aggregateArtifactBytes: authorityLimits.aggregateArtifactBytes ?? 1024 * 1024, + }; +} + +export function isLiveCaseApplicableToRoute(caseRecord: CaseRecord, route: LabRouteContext): boolean { + const req = caseRecord.requirements; + if (!req.inboundProtocols.includes(route.inboundProtocol) || !req.upstreamProtocols.includes(route.upstreamProtocol) || !req.surfaces.includes(route.surface)) return false; + if (req.platforms.length > 0 && !req.platforms.includes(process.platform)) return false; + if (!req.requiredClaims.every((claim) => (route.requiredClaims ?? []).includes(claim))) return false; + if (!req.requiredHarnessFeatures.every((feature) => (route.availableHarnessFeatures ?? []).includes(feature))) return false; + return true; +} + +function routePreconditionFailure(route: LabRouteContext, caseRecord: CaseRecord): string | null { + for (const precondition of caseRecord.requirements.routePreconditions) { + if (precondition === "lab_run_approval" && route.labRunApproval !== true) return "route_precondition_unmet:lab_run_approval"; + if (precondition === "allow_private_network" && route.allowPrivateNetwork !== true) return "route_precondition_unmet:allow_private_network"; + } + return null; +} + +function classifyWithFailureRules(rules: FailureRule[], signal: string): { classification: FailureClassification; secondaryCode: string; retryPolicy: FailureRule["retry"] | null } { + const rule = rules.find((row) => row.match.includes(signal)); + return rule ? { classification: rule.classification, secondaryCode: rule.secondaryCode ?? signal, retryPolicy: rule.retry } + : { classification: "inconclusive", secondaryCode: "unclassified", retryPolicy: null }; +} + +function pathForProtocol(protocol: string): string { + if (protocol === "openai-chat") return "/chat/completions"; + if (protocol === "anthropic-messages") return "/messages"; + return "/responses"; +} + +function chatObservation(body: string, status: number): NormalizedObservation { + const observation = emptyObservation(); + const output: unknown[] = []; + let text = ""; + let terminal = false; + const trimmed = body.trimStart(); + if (trimmed.startsWith("data:")) { + const toolParts = new Map(); + for (const frame of body.split(/\r?\n\r?\n/)) { + const line = frame.split(/\r?\n/).find((row) => row.startsWith("data:")); + if (!line) continue; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") { if (payload === "[DONE]") terminal = true; continue; } + let json: any; + try { json = JSON.parse(payload); } catch { continue; } + for (const choice of Array.isArray(json.choices) ? json.choices : []) { + if (typeof choice?.delta?.content === "string") text += choice.delta.content; + if (choice?.finish_reason != null) terminal = true; + for (const call of Array.isArray(choice?.delta?.tool_calls) ? choice.delta.tool_calls : []) { + const idx = Number.isInteger(call?.index) ? call.index : toolParts.size; + const prior = toolParts.get(idx) ?? { id: "", name: "", arguments: "" }; + if (typeof call?.id === "string") prior.id = call.id; + if (typeof call?.function?.name === "string") prior.name = call.function.name; + if (typeof call?.function?.arguments === "string") prior.arguments += call.function.arguments; + toolParts.set(idx, prior); + } + } + } + for (const row of [...toolParts.entries()].sort((a, b) => a[0] - b[0]).map(([, value]) => value)) { + output.push({ type: "function_call", call_id: row.id, name: row.name, arguments: row.arguments }); + } + } else { + const json = JSON.parse(body) as any; + const choice = Array.isArray(json.choices) ? json.choices[0] : undefined; + if (typeof choice?.message?.content === "string") text = choice.message.content; + terminal = choice?.finish_reason != null; + for (const call of Array.isArray(choice?.message?.tool_calls) ? choice.message.tool_calls : []) { + output.push({ type: "function_call", call_id: call.id, name: call.function?.name, arguments: call.function?.arguments }); + } + } + if (text) output.unshift({ type: "message", content: [{ type: "output_text", text }] }); + const json = { status: terminal ? "completed" : "incomplete", output }; + const events: NormalizedEvent[] = terminal ? [{ event: "response.completed", data: { response: json }, ordinal: 0 }] : []; + finalizeObservation(observation, events, json, status); + return observation; +} + +function responsesObservation(body: string, status: number): NormalizedObservation { + const observation = emptyObservation(); + const trimmed = body.trimStart(); + if (trimmed.startsWith("data:") || trimmed.startsWith("event:")) { + const events = normalizeSseBytes(new TextEncoder().encode(body), "openai-responses"); + finalizeObservation(observation, events, null, status); + return observation; + } + const json = JSON.parse(body) as Record; + finalizeObservation(observation, [], json, status); + const state = typeof json.status === "string" ? json.status : undefined; + if (state === "completed" || state === "failed" || state === "incomplete") { + setClientResponse(observation, { terminal: state === "completed" ? "completed" : state }); + } + return observation; +} + +function normalizeTransportObservation(route: LabRouteContext, response: { status: number; body: string }): NormalizedObservation { + const observation = route.upstreamProtocol === "openai-chat" ? chatObservation(response.body, response.status) : responsesObservation(response.body, response.status); + if (route.surface.startsWith("anthropic-") && observation.client.response.terminal === "completed") { + setClientResponse(observation, { terminal: "message_stop" }); + } + return observation; +} + +async function withTotalTimeout(timeoutMs: number, run: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live scenario total timeout")), timeoutMs); + try { + return await Promise.race([ + run(controller.signal), + new Promise((_, reject) => controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true })), + ]); + } finally { clearTimeout(timer); } +} + +export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabRouteContext, opts: LiveExecutorOptions = {}): Promise { + const diagnostics: string[] = []; + const startedAt = Date.now(); + let routeSubject: RouteSubjectV1 | undefined; + let executionAuthority: LiveExecutionAuthority = "none"; + let authority: CaseAuthority | undefined; + let activeCase = caseRecord; + let trustedExecutionStarted = false; + let failureRules: FailureRule[] = []; + const complete = (partial: Omit, retryPolicy: FailureRule["retry"] | null = null): LiveScenarioRunResult => { + const result: LiveScenarioRunResult = { ...partial, executionAuthority, startedAt, completedAt: Math.max(startedAt, Date.now()) }; + if (trustedExecutionStarted && authority && result.routeSubject) sealTrustedLiveResult(result, activeCase, authority, retryPolicy); + return result; + }; + try { + const environment = prepareLiveSandbox(opts.env); + authority = loadLiveCaseAuthority(); + const canonicalCase = authority.cases.find((row) => row.id === caseRecord.id); + if (!canonicalCase || jcsStringify(canonicalCase) !== jcsStringify(caseRecord)) { + throw new TransportError("harness_failure", "live scenario does not match canonical authority"); + } + activeCase = canonicalCase; + if (!isLiveCaseApplicableToRoute(activeCase, routeContext)) { + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: "inconclusive", secondaryCode: "scenario_inapplicable", assertionResults: [], diagnostics, routeSubject }); + } + const preconditionFailure = routePreconditionFailure(routeContext, activeCase); + if (preconditionFailure) { + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: "inconclusive", secondaryCode: preconditionFailure, assertionResults: [], diagnostics: [preconditionFailure], routeSubject }); + } + const expanded = expandLiveScenario(activeCase, authority); + failureRules = expanded.failureRules as FailureRule[]; + const limits = liveLimitsFromAuthority(expanded.executionLimits as Record); + const destination = await createLabDestination({ baseUrl: routeContext.baseUrl, allowPrivateNetwork: routeContext.allowPrivateNetwork, labRunApproval: routeContext.labRunApproval, resolve: opts.resolve, configDir: opts.configDir }); + routeSubject = buildRouteSubjectV1(routeContext, destination, opts.configDir); + const state = createSandboxResourceState(); + let observation: NormalizedObservation; + if (opts.routeExecutor) { + if (!isTrustedLabRouteExecutor(opts.routeExecutor)) throw new TransportError("untrusted_route_executor", "untrusted route executor capability"); + executionAuthority = "trusted_route"; + trustedExecutionStarted = true; + observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: activeCase.id, initiatingRequest: activeCase.initiatingRequest?.bytesUtf8, limits, signal, environment })); + } else if (opts.transport && activeCase.initiatingRequest) { + executionAuthority = "test_transport"; + const body = activeCase.initiatingRequest.bytesUtf8; + const inputBytes = new TextEncoder().encode(body).byteLength; + enforceSandboxLimits(state, limits, { requests: 1, inputBytes }); + const response = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.transport!.request({ method: "POST", path: pathForProtocol(routeContext.upstreamProtocol), body, signal })); + if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`); + if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429"); + if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); + if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`); + const outputBytes = new TextEncoder().encode(response.body).byteLength; + enforceSandboxLimits(state, limits, { outputBytes, outputTokens: Math.ceil(outputBytes / 4) }); + observation = normalizeTransportObservation(routeContext, response); + } else { + throw new TransportError("live_transport_required", activeCase.initiatingRequest ? "live transport or trusted route executor required" : "trusted route executor required"); + } + const assertionResults = evaluateAssertions(activeCase.assertions, observation); + const requiredFailures = assertionResults.filter((row) => row.required && !row.passed); + if (activeCase.expectedFailure) { + const matched = activeCase.expectedFailure.assertionIds.every((id) => assertionResults.find((row) => row.id === id)?.passed === true) && requiredFailures.length === 0; + if (matched) { + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: true, classification: activeCase.expectedFailure.expectedClass, secondaryCode: activeCase.expectedFailure.expectedCode, assertionResults, diagnostics, routeSubject }); + } + const mismatch = classifyWithFailureRules(failureRules, "required_assertion_failed"); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: mismatch.classification, secondaryCode: mismatch.secondaryCode, assertionResults, diagnostics, routeSubject }, mismatch.retryPolicy); + } + const passed = requiredFailures.length === 0; + const classified = passed ? { classification: "inconclusive" as FailureClassification, secondaryCode: "pass", retryPolicy: null } : classifyWithFailureRules(failureRules, "required_assertion_failed"); + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed, classification: passed ? "inconclusive" : classified.classification, secondaryCode: passed ? undefined : classified.secondaryCode, assertionResults, diagnostics, routeSubject }, passed ? null : classified.retryPolicy); + } catch (error) { + const diagnosticCode = error instanceof TransportError || error instanceof LabSandboxError || error instanceof LabDestinationError + ? error.code + : "execution_error"; + diagnostics.push(diagnosticCode); + let classified = classifyTransportError(error); + if (error instanceof LabSandboxError) classified = { classification: error.code === "harness_failure" ? "harness_failure" : "budget_exhausted", secondaryCode: error.code }; + if (error instanceof LabDestinationError) classified = { classification: error.code === "network_blocked" ? "network_failure" : "harness_failure", secondaryCode: error.code }; + const failureSignal = error instanceof TransportError || error instanceof LabSandboxError || error instanceof LabDestinationError ? error.code : "harness_failure"; + const retryPolicy = failureRules.find((rule) => rule.match.includes(failureSignal))?.retry ?? null; + return complete({ scenarioId: activeCase.id, suite: activeCase.suite, passed: false, classification: classified.classification, secondaryCode: classified.secondaryCode, assertionResults: [], diagnostics, routeSubject, transportError: error instanceof TransportError ? error.code : undefined }, retryPolicy); + } +} \ No newline at end of file diff --git a/src/lab/live/inert-tools.ts b/src/lab/live/inert-tools.ts new file mode 100644 index 000000000..8f6b33b07 --- /dev/null +++ b/src/lab/live/inert-tools.ts @@ -0,0 +1,56 @@ +export interface InertToolDefinition { + name: string; + kind: "function" | "custom"; + description?: string; + parameters?: Record; +} + +export interface InertToolResult { + output: string; +} + +const LAB_TOOLS: Record = { + lookup: { + name: "lookup", + kind: "function", + description: "Lab inert lookup tool", + parameters: { + type: "object", + properties: { q: { type: "string" } }, + required: ["q"], + }, + staticResult: { output: "RESULT" }, + }, + apply_patch: { + name: "apply_patch", + kind: "custom", + description: "Lab inert patch tool", + staticResult: { output: "PATCHED" }, + }, +}; + +export function listInertToolDefinitions(): InertToolDefinition[] { + return Object.values(LAB_TOOLS).map(({ staticResult: _r, ...tool }) => tool); +} + +export function validateInertToolArgs(name: string, args: unknown): void { + const tool = LAB_TOOLS[name]; + if (!tool) throw new Error(`unknown inert tool: ${name}`); + if (tool.kind === "function") { + if (!args || typeof args !== "object" || Array.isArray(args)) { + throw new Error("invalid function tool arguments"); + } + const record = args as Record; + if (typeof record.q !== "string") { + throw new Error("lookup requires string q"); + } + } +} + +/** Execute an inert lab tool — validate args and return static results only. */ +export function executeInertTool(name: string, args: unknown): InertToolResult { + validateInertToolArgs(name, args); + const tool = LAB_TOOLS[name]; + if (!tool) throw new Error(`unknown inert tool: ${name}`); + return { ...tool.staticResult }; +} diff --git a/src/lab/live/manifest.ts b/src/lab/live/manifest.ts new file mode 100644 index 000000000..c62bee96a --- /dev/null +++ b/src/lab/live/manifest.ts @@ -0,0 +1,85 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { fixtureDigest } from "../digest"; +import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule } from "../conformance/types"; +import { CL03_LIVE_SUITES, SYNTHETIC_MARKER } from "../conformance/types"; + +const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +const AUTHORITY_FILE = "024_live_v1_cases.json"; +const RUNTIME_AUTHORITY = join(MODULE_DIR, "..", "conformance", "fixtures", "live-v1-cases.json"); +const REPO_AUTHORITY = join(MODULE_DIR, "..", "..", "..", "devlog", "_plan", "260807_compatibility_lab", AUTHORITY_FILE); + +function readAuthority(path: string): CaseAuthority { return JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; } + +/** During source-tree execution, fail closed unless the runtime copy is byte-identical to normative 024. */ +export function assertLiveAuthorityMirror(): void { + if (!existsSync(REPO_AUTHORITY)) return; + const runtimeBytes = readFileSync(RUNTIME_AUTHORITY); + const normativeBytes = readFileSync(REPO_AUTHORITY); + if (!runtimeBytes.equals(normativeBytes)) throw new Error("harness_failure: live authority runtime copy drift"); +} + +export function loadLiveCaseAuthority(): CaseAuthority { + assertLiveAuthorityMirror(); + const raw = readAuthority(RUNTIME_AUTHORITY); + validateLiveAuthority(raw); + return raw; +} + +export function discoverLiveScenarios(authority: CaseAuthority, suites: readonly string[] = CL03_LIVE_SUITES): CaseRecord[] { + return authority.cases.filter((row) => suites.includes(row.suite)); +} + +export function expandLiveScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { + const defaults = authority.manifestDefaults; + const fixtures = caseRecord.initiatingRequest ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] : [fixtureRef(caseRecord.fixture, authority)]; + return { + schemaVersion: authority.schemaVersion, id: caseRecord.id, version: defaults.version, + suite: { id: caseRecord.suite, version: defaults.suiteVersion, evidenceLayer: defaults.evidenceLayer }, + evidenceLayer: defaults.evidenceLayer, capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, requirements: caseRecord.requirements, + fixtures, executionLimits: defaults.executionLimits, executionMode: defaults.executionMode, assertions: caseRecord.assertions, + ...(caseRecord.expectedFailure ? { expectedFailure: caseRecord.expectedFailure } : {}), + failureRules: expandLiveFailureRules(caseRecord, authority), artifactPolicy: defaults.artifactPolicy, freshness: defaults.freshness, + }; +} + +function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): Record { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + return { id: fixture.id, role: fixture.role, mediaType: fixture.mediaType, digest: fixture.digest, byteLength: bytes.byteLength, + syntheticMarker: SYNTHETIC_MARKER, provenance: { kind: "lab_authored", authority: AUTHORITY_FILE, sourceCommit: authority.sourceCommit } }; +} + +function expandLiveFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { + const setName = authority.manifestDefaults.failureRuleSet; + const ruleSet = authority.failureRuleSets[setName]; + if (!Array.isArray(ruleSet)) throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); + const base = [...ruleSet]; + if (!caseRecord.expectedFailure) return base; + const template = authority.expectedFailureRuleTemplate; + const controlRule: FailureRule = { id: template.id, match: [...template.match], classification: caseRecord.expectedFailure.expectedClass as FailureClassification, + secondaryCode: caseRecord.expectedFailure.expectedCode, verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", + retry: template.retry, expected: template.expected }; + const idx = base.findIndex((row) => row.id === "required-assertion"); + if (idx >= 0) base.splice(idx, 0, controlRule); else base.push(controlRule); + return base; +} + +function validateLiveAuthority(authority: CaseAuthority): void { + if (authority.schemaVersion !== 1 || authority.manifestDefaults.evidenceLayer !== "live_route_compatibility") throw new Error("invalid live authority"); + if (!authority.sourceCommit || !Array.isArray(authority.cases) || authority.cases.length !== 10) throw new Error("invalid live authority case set"); + const ids = new Set(); + for (const caseRecord of authority.cases) { + if (ids.has(caseRecord.id)) throw new Error(`duplicate live scenario ${caseRecord.id}`); + ids.add(caseRecord.id); + if (!caseRecord.id.split(".").includes("live")) throw new Error(`non-live scenario in CL-03 authority: ${caseRecord.id}`); + const bytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + if (fixtureDigest(bytes) !== caseRecord.fixture.digest) throw new Error(`${caseRecord.id}: fixture digest mismatch`); + if (caseRecord.initiatingRequest) { + const initBytes = new TextEncoder().encode(caseRecord.initiatingRequest.bytesUtf8); + if (fixtureDigest(initBytes) !== caseRecord.initiatingRequest.digest) throw new Error(`${caseRecord.id}: initiatingRequest digest mismatch`); + } + expandLiveScenario(caseRecord, authority); + } +} diff --git a/src/lab/live/mcp-loopback.ts b/src/lab/live/mcp-loopback.ts new file mode 100644 index 000000000..3d08c61ca --- /dev/null +++ b/src/lab/live/mcp-loopback.ts @@ -0,0 +1,57 @@ +export interface McpStubTool { + namespace: string; + name: string; + inputSchema?: Record; +} + +export interface McpStubCall { + namespace: string; + name: string; + arguments: Record; +} + +export interface McpStubResult { + content: Array<{ type: "text"; text: string }>; +} + +const tools = new Map(); +const invocations: McpStubCall[] = []; + +function toolKey(namespace: string, name: string): string { + return `${namespace}::${name}`; +} + +/** Register a lab-owned MCP stub tool (in-memory only). */ +export function registerMcpStubTool(tool: McpStubTool): void { + tools.set(toolKey(tool.namespace, tool.name), { ...tool }); +} + +export function clearMcpStub(): void { + tools.clear(); + invocations.length = 0; +} + +export function listMcpStubTools(): McpStubTool[] { + return [...tools.values()]; +} + +/** Invoke the in-memory MCP stub with fixed pure results. */ +export function invokeMcpStub( + namespace: string, + name: string, + args: Record, + result: McpStubResult, +): McpStubResult { + const key = toolKey(namespace, name); + if (!tools.has(key)) throw new Error(`MCP stub tool not registered: ${namespace}/${name}`); + invocations.push({ namespace, name, arguments: { ...args } }); + return result; +} + +export function mcpStubInvocations(): readonly McpStubCall[] { + return invocations; +} + +export function wireName(namespace: string, name: string): string { + return `mcp__${namespace}__${name}`; +} diff --git a/src/lab/live/runner.ts b/src/lab/live/runner.ts new file mode 100644 index 000000000..ed4e2ae99 --- /dev/null +++ b/src/lab/live/runner.ts @@ -0,0 +1,19 @@ +import { CL03_LIVE_SUITES } from "../conformance/types"; +import { isLiveCaseApplicableToRoute, runLiveScenario, type LiveExecutorOptions } from "./executor"; +import { discoverLiveScenarios, loadLiveCaseAuthority } from "./manifest"; +import type { LabRouteContext, LiveScenarioRunResult } from "./types"; + +export interface LiveRunSummary { total: number; passed: number; failed: number; results: LiveScenarioRunResult[] } + +export async function runLiveSuite(routeContext: LabRouteContext, suites: readonly string[] = CL03_LIVE_SUITES, opts: LiveExecutorOptions = {}): Promise { + const authority = loadLiveCaseAuthority(); + const scenarios = discoverLiveScenarios(authority, suites).filter((scenario) => isLiveCaseApplicableToRoute(scenario, routeContext)); + const results: LiveScenarioRunResult[] = []; + for (const scenario of scenarios) results.push(await runLiveScenario(scenario, routeContext, opts)); + const passed = results.filter((row) => row.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export function listLiveScenarioIds(suites: readonly string[] = CL03_LIVE_SUITES): string[] { + return discoverLiveScenarios(loadLiveCaseAuthority(), suites).map((scenario) => scenario.id); +} diff --git a/src/lab/live/sandbox.ts b/src/lab/live/sandbox.ts new file mode 100644 index 000000000..7399a31c9 --- /dev/null +++ b/src/lab/live/sandbox.ts @@ -0,0 +1,61 @@ +import type { LiveRunConfig } from "./types"; + +const PROXY_ENV_VARS = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"] as const; +const ALLOWED_ENV = { TZ: "UTC", NO_COLOR: "1" } as const; + +export class LabSandboxError extends Error { + override readonly name = "LabSandboxError"; + constructor(message: string, readonly code: string) { super(message); } +} + +export function rejectProxyEnvironment(env: NodeJS.ProcessEnv = process.env): void { + for (const name of PROXY_ENV_VARS) { + const value = env[name]; + if (value !== undefined && value !== "") throw new LabSandboxError(`proxy environment variable ${name} is forbidden`, "harness_failure"); + } +} + +export function labSandboxEnvironment(): Readonly> { return Object.freeze({ ...ALLOWED_ENV }); } + +export interface SandboxResourceState { + requests: number; + inputBytes: number; + outputBytes: number; + outputTokens: number; + toolCalls: number; + artifacts: number; + artifactBytes: number; + childProcesses: number; +} + +export function createSandboxResourceState(): SandboxResourceState { + return { requests: 0, inputBytes: 0, outputBytes: 0, outputTokens: 0, toolCalls: 0, artifacts: 0, artifactBytes: 0, childProcesses: 0 }; +} + +/** + * Counter-based ceilings enforce the resources observable in this process. The hard resident-memory + * boundary is NOT inferred from the hosting Bun test/server process RSS; a trusted exact-route + * executor must attest/enforce that isolated boundary before its result is evidence-eligible. + */ +export function enforceSandboxLimits(state: SandboxResourceState, limits: LiveRunConfig, delta: Partial = {}): void { + const next: SandboxResourceState = { + requests: state.requests + (delta.requests ?? 0), inputBytes: state.inputBytes + (delta.inputBytes ?? 0), + outputBytes: state.outputBytes + (delta.outputBytes ?? 0), outputTokens: state.outputTokens + (delta.outputTokens ?? 0), + toolCalls: state.toolCalls + (delta.toolCalls ?? 0), artifacts: state.artifacts + (delta.artifacts ?? 0), + artifactBytes: state.artifactBytes + (delta.artifactBytes ?? 0), childProcesses: state.childProcesses + (delta.childProcesses ?? 0), + }; + if (next.childProcesses > limits.maxChildProcesses) throw new LabSandboxError("child process limit exceeded", "child_process_limit"); + if (next.requests > limits.maxRequests) throw new LabSandboxError("request limit exceeded", "request_limit"); + if (next.inputBytes > limits.maxInputBytes) throw new LabSandboxError("input byte limit exceeded", "input_byte_limit"); + if (next.outputBytes > limits.maxOutputBytes) throw new LabSandboxError("output byte limit exceeded", "output_byte_limit"); + if (next.outputTokens > limits.maxOutputTokens) throw new LabSandboxError("output token limit exceeded", "output_token_limit"); + if (next.toolCalls > limits.maxToolCalls) throw new LabSandboxError("tool call limit exceeded", "tool_call_limit"); + if (next.artifacts > limits.maxArtifacts) throw new LabSandboxError("artifact limit exceeded", "artifact_byte_limit"); + if (next.artifactBytes > limits.aggregateArtifactBytes) throw new LabSandboxError("aggregate artifact limit exceeded", "artifact_byte_limit"); + Object.assign(state, next); +} + +export function prepareLiveSandbox(env: NodeJS.ProcessEnv = process.env): Readonly> { + rejectProxyEnvironment(env); + return labSandboxEnvironment(); +} diff --git a/src/lab/live/suite-manifest.ts b/src/lab/live/suite-manifest.ts new file mode 100644 index 000000000..203bbe650 --- /dev/null +++ b/src/lab/live/suite-manifest.ts @@ -0,0 +1,41 @@ +import { scenarioManifestDigest, suiteManifestDigest } from "../digest"; +import type { CaseAuthority, CaseRecord, VerificationRole } from "../conformance/types"; +import { expandLiveScenario } from "./manifest"; + +export interface LiveSuiteScenarioRefV1 { id: string; version: string; role: VerificationRole; manifestDigest: string } +export interface LiveSuiteManifestV1 { + schemaVersion: 1; id: string; version: string; evidenceLayer: string; capability: string; + assertionDslVersion: string; evidenceSchemaVersion: string; freshness: { maxAgeMs: number | null }; + contradictionRule: string; scenarios: LiveSuiteScenarioRefV1[]; verificationRule: string; +} + +function casesForSuiteManifest(suiteId: string, authority: CaseAuthority): { own: CaseRecord[]; referenced: CaseRecord[] } { + const own = authority.cases.filter((row) => row.suite === suiteId); + if (own.length === 0) throw new Error(`unknown live suite ${suiteId}`); + if (suiteId !== "codex-core") return { own, referenced: own }; + const prerequisite = authority.cases.find((row) => row.id === "responses-core.live.basic-turn"); + if (!prerequisite) throw new Error("harness_failure: codex live prerequisite missing"); + return { own, referenced: [...own, prerequisite] }; +} + +export function expandLiveSuiteManifest(suiteId: string, authority: CaseAuthority): LiveSuiteManifestV1 { + const { own, referenced } = casesForSuiteManifest(suiteId, authority); + const defaults = authority.manifestDefaults; + const scenarios = referenced.map((caseRecord) => ({ + id: caseRecord.id, version: String(defaults.version), role: caseRecord.verificationRole ?? defaults.verificationRole, + manifestDigest: scenarioManifestDigest(expandLiveScenario(caseRecord, authority)), + })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0); + return { + schemaVersion: 1, id: suiteId, version: String(defaults.suiteVersion), evidenceLayer: defaults.evidenceLayer, + capability: own[0]!.capability, assertionDslVersion: authority.assertionDslVersion, evidenceSchemaVersion: authority.evidenceSchemaVersion, + freshness: defaults.freshness ?? { maxAgeMs: null }, contradictionRule: "newest-required-observation-v1", scenarios, + verificationRule: "all-applicable-required-pass-v1", + }; +} + +export function liveSuiteManifestObjectForCase(caseRecord: CaseRecord, authority: CaseAuthority): Record { + return expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record; +} +export function liveSuiteManifestDigestForCase(caseRecord: CaseRecord, authority: CaseAuthority): string { + return suiteManifestDigest(expandLiveSuiteManifest(caseRecord.suite, authority) as unknown as Record); +} diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts new file mode 100644 index 000000000..90b0b8bea --- /dev/null +++ b/src/lab/live/transport.ts @@ -0,0 +1,116 @@ +import type { FailureClassification } from "../conformance/types"; +import { assertLeaseScope, LabCredentialError } from "./credential-lease"; +import type { + LabCredentialLeaseV1, LabDestinationV1, LabPinnedSender, LabTransport, LabTransportRequest, + LabTransportResponse, LiveRunConfig, TransportErrorCode, +} from "./types"; + +export interface MockTransportEntry { + matchPath?: string | RegExp; + status: number; + headers?: Record; + body: string; + error?: TransportErrorCode; +} +export interface MockTransportOptions { entries?: MockTransportEntry[]; onRequest?: (req: LabTransportRequest) => void } + +/** Test seam only. No live runner path creates this implicitly. */ +export function createMockTransport(opts: MockTransportOptions = {}): LabTransport { + let callIndex = 0; + return { + async request(req): Promise { + opts.onRequest?.(req); + const entry = opts.entries?.[callIndex] ?? opts.entries?.[opts.entries.length - 1]; + callIndex += 1; + if (!entry) throw new TransportError("harness_failure", "no mock transport entry"); + if (entry.error) throw new TransportError(entry.error, entry.error); + if (entry.matchPath) { + const ok = typeof entry.matchPath === "string" ? req.path.includes(entry.matchPath) : entry.matchPath.test(req.path); + if (!ok) throw new TransportError("harness_failure", "mock path mismatch"); + } + if (entry.status >= 300 && entry.status < 400) throw new TransportError("redirect_blocked", "redirect response"); + return { status: entry.status, headers: { "content-type": "application/json", ...(entry.headers ?? {}) }, body: entry.body }; + }, + }; +} + +export interface PinnedTransportOptions { + destination: LabDestinationV1; + lease: LabCredentialLeaseV1; + sender: LabPinnedSender; + limits: LiveRunConfig; + transportId?: string; +} + +function assertRelativeRequestPath(path: string): void { + if (!path.startsWith("/") || path.startsWith("//") || path.includes("://") || /[\r\n]/.test(path)) { + throw new TransportError("harness_failure", "request path attempted to widen destination"); + } +} + +function selectedPinnedAddress(destination: LabDestinationV1): { address: string; family: 4 | 6 } { + const row = destination.addresses.find((a) => a.family === 4) ?? destination.addresses[0]; + if (!row) throw new TransportError("harness_failure", "approved destination has no address"); + return { address: row.address, family: row.family }; +} + +/** Uses only the frozen approved address set. Secret injection is owned by the trusted sender, outside Lab. */ +export function createPinnedTransport(opts: PinnedTransportOptions): LabTransport { + const transportId = opts.transportId ?? "default"; + return { + async request(req): Promise { + assertRelativeRequestPath(req.path); + assertLeaseScope(opts.lease, opts.destination, transportId); + const inputBytes = new TextEncoder().encode(req.body ?? "").byteLength; + if (inputBytes > opts.limits.maxInputBytes) throw new TransportError("input_byte_limit", "request exceeds input byte budget"); + opts.lease.consume(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live request total timeout")), opts.limits.totalTimeoutMs); + const onAbort = () => controller.abort(req.signal?.reason); + req.signal?.addEventListener("abort", onAbort, { once: true }); + try { + const response = await opts.sender(opts.lease, opts.destination, selectedPinnedAddress(opts.destination), req, controller.signal, opts.limits); + const outputBytes = new TextEncoder().encode(response.body).byteLength; + if (outputBytes > opts.limits.maxOutputBytes) throw new TransportError("output_byte_limit", "response exceeds output byte budget"); + if (response.status >= 300 && response.status < 400) throw new TransportError("redirect_blocked", "redirect response"); + if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`); + if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429"); + if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); + if (response.status >= 500) throw new TransportError("provider_transient", `HTTP ${response.status}`); + return response; + } catch (error) { + if (error instanceof LabCredentialError) { + throw new TransportError(error.code === "budget_exhausted" ? "request_limit" : "harness_failure", error.message); + } + throw error; + } finally { + clearTimeout(timer); + req.signal?.removeEventListener("abort", onAbort); + } + }, + }; +} + +export class TransportError extends Error { + override readonly name = "TransportError"; + constructor(readonly code: TransportErrorCode, message: string) { super(message); } +} + +export function classifyTransportError(error: unknown): { classification: FailureClassification; secondaryCode: string } { + const code = error instanceof TransportError ? error.code : undefined; + switch (code) { + case "auth_blocked": return { classification: "authentication_blocked", secondaryCode: code }; + case "quota_blocked": return { classification: "quota_blocked", secondaryCode: code }; + case "network_blocked": return { classification: "network_failure", secondaryCode: code }; + case "region_blocked": return { classification: "region_blocked", secondaryCode: code }; + case "provider_transient": return { classification: "provider_transient", secondaryCode: code }; + case "connect_timeout": case "first_byte_timeout": case "inactivity_timeout": case "total_timeout": + return { classification: "timeout", secondaryCode: code }; + case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit": + case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit": + return { classification: "budget_exhausted", secondaryCode: code }; + case "live_transport_required": case "untrusted_route_executor": case "redirect_blocked": case "destination_mismatch": case "host_sni_mismatch": case "harness_failure": + return { classification: "harness_failure", secondaryCode: code }; + default: return { classification: "harness_failure", secondaryCode: "execution_error" }; + } +} diff --git a/src/lab/live/types.ts b/src/lab/live/types.ts new file mode 100644 index 000000000..8a30cfc79 --- /dev/null +++ b/src/lab/live/types.ts @@ -0,0 +1,197 @@ +import type { FailureClassification, NormalizedObservation } from "../conformance/types"; +import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types"; + +export interface LabDestinationV1 { + readonly scheme: "http" | "https"; + readonly host: string; + readonly port: number; + readonly basePath: string; + readonly sniHost: string; + readonly addresses: ReadonlyArray<{ readonly address: string; readonly family: 4 | 6 }>; + readonly privateNetwork: boolean; + readonly fingerprint: string; +} + +export type LabBehaviorSource = + | "request" + | "model_override" + | "provider_config" + | "registry_runtime_default" + | "generated_model_metadata" + | "global_config" + | "adapter_default" + | "lab_forced"; + +export interface LabBehaviorValue { + source: LabBehaviorSource; + value: unknown; +} + +/** Authoritative effective values emitted by the production route/model/adapter resolver. */ +export type LabBehaviorValues = Record; + +export interface LabRouteContext { + providerId: string; + /** Config-owner identity only; HMACed before it enters evidence. */ + providerInstanceKey: string; + clientModelId: string; + upstreamModelId: string; + effectiveAdapter: string; + inboundProtocol: string; + upstreamProtocol: string; + surface: string; + baseUrl: string; + /** Generated compatibility hash, not package marketing version. */ + opencodexCompatibilityVersion: string; + /** Closed, effective behavior inputs from the production resolver. */ + behaviorValues: LabBehaviorValues; + allowPrivateNetwork?: boolean; + labRunApproval?: boolean; + requiredClaims?: string[]; + availableHarnessFeatures?: string[]; + dependencies?: RouteDependencyV1[]; +} + +export interface LiveRunConfig { + totalTimeoutMs: number; + connectTimeoutMs: number; + firstByteTimeoutMs: number; + inactivityTimeoutMs: number; + maxRequests: number; + maxInputBytes: number; + maxOutputBytes: number; + maxOutputTokens: number; + maxToolCalls: number; + maxMemoryBytes: number; + maxChildProcesses: number; + maxArtifacts: number; + perArtifactBytes: number; + aggregateArtifactBytes: number; +} + +export type DnsResolver = (hostname: string) => Promise>; + +export type TransportErrorCode = + | "auth_blocked" + | "quota_blocked" + | "network_blocked" + | "region_blocked" + | "provider_transient" + | "redirect_blocked" + | "destination_mismatch" + | "host_sni_mismatch" + | "connect_timeout" + | "first_byte_timeout" + | "inactivity_timeout" + | "total_timeout" + | "request_limit" + | "input_byte_limit" + | "output_byte_limit" + | "output_token_limit" + | "tool_call_limit" + | "artifact_byte_limit" + | "memory_limit" + | "child_process_limit" + | "live_transport_required" + | "untrusted_route_executor" + | "harness_failure"; + +export interface LabTransportRequest { + method: "GET" | "POST"; + path: string; + headers?: Record; + body?: string; + signal?: AbortSignal; +} + +export interface LabTransportResponse { + status: number; + headers: Record; + body: string; +} + +export type TransportRequest = LabTransportRequest; +export type TransportResponse = LabTransportResponse; + +export interface LabTransport { + request(req: LabTransportRequest): Promise; +} + +export const LAB_CREDENTIAL_LEASE = Symbol("LabCredentialLeaseV1"); + +/** Opaque capability. Scope metadata and secret material are deliberately not public fields. */ +export interface LabCredentialLeaseV1 { + readonly [LAB_CREDENTIAL_LEASE]: true; + readonly remainingRequests: number; + consume(): void; +} + +export interface LabPinnedAddress { + address: string; + family: 4 | 6; +} + +/** Trusted provider/credential transport seam. Implementations own secret injection outside Lab. */ +export type LabPinnedSender = ( + lease: LabCredentialLeaseV1, + destination: LabDestinationV1, + pinned: LabPinnedAddress, + request: LabTransportRequest, + signal: AbortSignal, + limits: LiveRunConfig, +) => Promise; + +export interface LabRouteExecutorInput { + routeContext: LabRouteContext; + destination: LabDestinationV1; + routeSubject: RouteSubjectV1; + scenarioId: string; + initiatingRequest?: string; + limits: LiveRunConfig; + signal: AbortSignal; + environment: Readonly>; +} + +/** Trusted exact-route execution function. The wrapper capability is issued only by the host integration. */ +export type LabRouteExecutor = (input: LabRouteExecutorInput) => Promise; + +export const REQUIRED_LAB_SANDBOX_BOUNDARIES = [ + "destination_pinning", + "credential_binding", + "wall_clock", + "connect_timeout", + "first_byte_timeout", + "inactivity_timeout", + "request_budget", + "input_byte_budget", + "output_byte_budget", + "output_token_budget", + "tool_call_budget", + "memory_limit", + "child_process_limit", + "artifact_budget", +] as const; +export type LabSandboxBoundary = typeof REQUIRED_LAB_SANDBOX_BOUNDARIES[number]; + +/** Opaque host-issued capability; runtime trust is established outside Lab. */ +export interface TrustedLabRouteExecutor { + execute(input: LabRouteExecutorInput): Promise; + readonly enforcedBoundaries: readonly LabSandboxBoundary[]; +} + +export type LiveExecutionAuthority = "trusted_route" | "test_transport" | "none"; + +export interface LiveScenarioRunResult { + scenarioId: string; + suite: string; + startedAt: number; + completedAt: number; + passed: boolean; + classification: FailureClassification; + secondaryCode?: string; + assertionResults: import("../conformance/types").AssertionResult[]; + diagnostics: string[]; + routeSubject?: RouteSubjectV1; + transportError?: TransportErrorCode; + executionAuthority: LiveExecutionAuthority; +} diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 606b95b86..048e92a73 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -139,6 +139,11 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { switch (result.classification) { case "timeout": case "budget_exhausted": + case "authentication_blocked": + case "quota_blocked": + case "region_blocked": + case "network_failure": + case "provider_transient": return "blocked"; case "inconclusive": case "harness_failure": diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts new file mode 100644 index 000000000..c56f55191 --- /dev/null +++ b/src/lab/observe/from-live.ts @@ -0,0 +1,100 @@ +// CL-03 live result -> CL-02 immutable observation persistence. +import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; +import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PRODUCER_VERSION, OBSERVATION_LIMIT_NAMES, type ObservationOutcome } from "../constants"; +import { fixtureDigest, scenarioManifestDigest, subjectIdForSubject, suiteManifestDigest } from "../digest"; +import type { FailureRecordV1, ObservationEvent } from "../events/types"; +import { assignEventId } from "../events/validate"; +import { appendLabEvent } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import type { CaseAuthority, CaseRecord } from "../conformance/types"; +import { trustedLiveResultRetryable } from "../live/executor"; +import { expandLiveScenario } from "../live/manifest"; +import { liveSuiteManifestObjectForCase } from "../live/suite-manifest"; +import type { LiveScenarioRunResult } from "../live/types"; + +export interface PersistLiveOptions { configDir?: string; recordedAt?: number; startedAt?: number; completedAt?: number; producerVersion?: string; artifactStore?: ArtifactStore } +export interface PersistedLiveObservation { event: ObservationEvent; ledgerPath: string } + +function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcome { + if (result.passed) return "pass"; + switch (result.classification) { + case "timeout": + case "budget_exhausted": + case "authentication_blocked": + case "quota_blocked": + case "region_blocked": + case "network_failure": + case "provider_transient": + return "blocked"; + case "protocol_failure": + case "capability_failure": + case "behavioral_failure": + return "fail"; + case "inconclusive": + case "harness_failure": + return "inconclusive"; + } +} + +function failureFromLiveResult(result: LiveScenarioRunResult, retryable: boolean): FailureRecordV1 | undefined { + if (result.passed || result.classification === "inconclusive") return undefined; + const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" + ? "harness" + : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) + ? "environment" + : "route"; + return { + class: result.classification, + code: result.secondaryCode ?? result.classification, + retryable, + attribution, + }; +} + +function requireExecutionTimes(result: LiveScenarioRunResult, opts: PersistLiveOptions): { startedAt: number; completedAt: number } { + const startedAt = opts.startedAt ?? result.startedAt; const completedAt = opts.completedAt ?? result.completedAt; + if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt) || startedAt < 0 || completedAt < startedAt) throw new Error("invalid persisted live execution timestamps"); + return { startedAt, completedAt }; +} + +export function observationFromLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): { event: ObservationEvent; artifacts: ReturnType[] } { + const retryable = trustedLiveResultRetryable(result, caseRecord, authority); + if (!result.routeSubject) throw new Error("live evidence without an exact RouteSubjectV1 is not persistable"); + const { startedAt, completedAt } = requireExecutionTimes(result, opts); + const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const recordedAt = opts.recordedAt ?? completedAt; + const expandedScenario = expandLiveScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); + const suiteExpanded = liveSuiteManifestObjectForCase(caseRecord, authority); const suiteDigest = suiteManifestDigest(suiteExpanded); + const fixtureDigests: string[] = []; const artifacts: ReturnType[] = []; + const putFixture = (fixture: CaseRecord["fixture"]) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); const digest = fixtureDigest(bytes); fixtureDigests.push(digest); + artifacts.push(store.put({ artifactClass: "fixture", payload: bytes, expectedDigest: digest, mediaType: fixture.mediaType })); + }; + putFixture(caseRecord.fixture); if (caseRecord.initiatingRequest) putFixture(caseRecord.initiatingRequest); + artifacts.push(store.put({ artifactClass: "scenario_manifest", payload: expandedScenario, expectedDigest: scenarioDigest })); + artifacts.push(store.put({ artifactClass: "suite_manifest", payload: suiteExpanded, expectedDigest: suiteDigest })); + artifacts.push(store.put({ artifactClass: "assertion_report", payload: { scenarioId: result.scenarioId, passed: result.passed, classification: result.classification, + assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, observedSummary: row.observedSummary, reason: row.reason })) } })); + const subject = result.routeSubject; const subjectId = subjectIdForSubject(subject); const authorityLimits = authority.manifestDefaults.executionLimits; + const limits: Record = {}; for (const key of OBSERVATION_LIMIT_NAMES) if (key in authorityLimits) limits[key] = authorityLimits[key] ?? null; + const failure = failureFromLiveResult(result, retryable); + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, + evidenceLayer: "live_route_compatibility" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), scenarioManifestDigest: scenarioDigest, + suiteId: caseRecord.suite, suiteVersion: String(authority.manifestDefaults.suiteVersion), suiteManifestDigest: suiteDigest, fixtureDigests, subject, subjectId, + startedAt, completedAt, executionMode: "live" as const, attempt: 1, limits, outcome: outcomeFromLiveResult(result), + assertions: result.assertionResults.map((row) => ({ id: row.id, operator: row.operator, required: row.required, passed: row.passed, expectedSummary: "see_assertion_report", observedSummary: row.observedSummary.slice(0, 512), ...(row.reason ? { reason: row.reason } : {}) })), + ...(caseRecord.expectedFailure ? { expectedFailure: { ...caseRecord.expectedFailure } } : {}), + environment: { runtime: { platform: process.platform, arch: process.arch, bunVersion: process.versions.bun ?? Bun.version } }, artifactRefs: artifacts, + ...(failure ? { failure } : {}), + }; + return { event: assignEventId(eventWithoutId) as ObservationEvent, artifacts }; + } finally { if (ownsStore) store.close(); } +} + +export function persistLiveResult(result: LiveScenarioRunResult, caseRecord: CaseRecord, authority: CaseAuthority, opts: PersistLiveOptions = {}): PersistedLiveObservation { + const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { const { event } = observationFromLiveResult(result, caseRecord, authority, { ...opts, artifactStore: store }); appendLabEvent(paths.ledgerPath, event); return { event, ledgerPath: paths.ledgerPath }; } + finally { if (ownsStore) store.close(); } +} \ No newline at end of file diff --git a/src/lab/paths.ts b/src/lab/paths.ts index d54e11f4d..068db3203 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -34,6 +34,11 @@ export function labExportDir(configDir = getConfigDir()): string { return join(labRoot(configDir), "export"); } +/** Opaque per-installation salt for local fingerprinting (never exported as evidence). */ +export function labInstallationSaltPath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "installation-salt.bin"); +} + /** Ensure lab directories exist with restrictive permissions where the platform allows. */ export function ensureLabDirs(configDir = getConfigDir()): { root: string; diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index cafe66cd6..db0bc683e 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -166,6 +166,16 @@ export function resolveClaimStates( return { states, corruptions }; } +function supportedClaimsForSubject(claimStates: Map, subjectId: string): string[] { + const supported = new Set(); + for (const state of claimStates.values()) { + const claim = state.current; + if (!claim || state.corruption || state.unusable) continue; + if (claim.subjectId === subjectId && claim.polarity === "supported") supported.add(claim.capability); + } + return [...supported].sort(); +} + /** * CL-02 verdict projection primitives with frozen CL-00 verification semantics. */ @@ -232,6 +242,9 @@ export function projectVerdicts( projectObservationGroup(key, ordered, asOf, suiteManifest, { loadScenarioManifest: opts.loadScenarioManifest, loadScenarioRequirements: opts.loadScenarioRequirements, + routeSupportedClaims: key.evidenceLayer === "live_route_compatibility" + ? supportedClaimsForSubject(claimStates, key.subjectId) + : undefined, }), ); } @@ -296,6 +309,7 @@ function projectObservationGroup( opts: { loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + routeSupportedClaims?: readonly string[]; } = {}, ): DerivedVerdict { const contributing: string[] = []; @@ -365,6 +379,34 @@ function projectObservationGroup( notes.push("incomplete_required_coverage"); } } + } else if (key.evidenceLayer === "live_route_compatibility" && newestCurrent?.executionMode === "live") { + if (!suiteManifest) { + verdict = "PROBED"; + notes.push("suite_manifest_unavailable"); + } else { + const evaluation = evaluateAllApplicableRequiredPassV1( + suiteManifest, + ordered, + newestCurrent.executionMode, + { + subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, + routeSupportedClaims: opts.routeSupportedClaims, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.applicableRequiredScenarioIds.length === 0) { + verdict = "UNKNOWN"; + } else if (evaluation.canVerify) { + verdict = "VERIFIED"; + notes.push("all-applicable-required-pass-v1"); + } else { + verdict = "PROBED"; + notes.push("incomplete_required_coverage"); + } + } } else { verdict = "PROBED"; } diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 806a345e0..ac48e342d 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,4 +1,4 @@ -import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; +import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1 } from "../events/types"; import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; import type { VerificationRole } from "../conformance/types"; @@ -18,6 +18,7 @@ export interface ScenarioRequirements { inboundProtocols?: string[]; upstreamProtocols?: string[]; surfaces?: string[]; + requiredClaims?: string[]; freshness?: { maxAgeMs: number | null }; } @@ -51,6 +52,25 @@ function scenarioApplicableToRequirements( ); } +/** Applicability for live_route_compatibility using exact RouteSubjectV1 plus validated claim state. */ +export function routeSubjectApplicableToRequirements( + requirements: ScenarioRequirements, + subject: RouteSubjectV1, + supportedClaims: readonly string[], +): boolean { + const inbound = requirements.inboundProtocols ?? []; + const upstream = requirements.upstreamProtocols ?? []; + const surfaces = requirements.surfaces ?? []; + const requiredClaims = requirements.requiredClaims ?? []; + const claimsOk = requiredClaims.every((claim) => supportedClaims.includes(claim)); + return ( + inbound.includes(subject.inboundProtocol) && + upstream.includes(subject.upstreamProtocol) && + surfaces.includes(subject.surface) && + claimsOk + ); +} + function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } @@ -79,9 +99,11 @@ function scenarioContractFromManifest( const upstreamProtocols = parseStringArray(row.upstreamProtocols); const surfaces = parseStringArray(row.surfaces); if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; + const requiredClaims = row.requiredClaims === undefined ? [] : parseStringArray(row.requiredClaims); + if (!requiredClaims) return null; const freshness = parseFreshness(scenarioManifest.freshness); if (!freshness) return null; - return { inboundProtocols, upstreamProtocols, surfaces, freshness }; + return { inboundProtocols, upstreamProtocols, surfaces, requiredClaims, freshness }; } function effectiveMaxAgeMs( @@ -117,7 +139,9 @@ export function evaluateAllApplicableRequiredPassV1( observations: ObservationEvent[], executionMode: ExecutionMode, opts: { - subject?: ProtocolSubjectV1; + subject?: ProtocolSubjectV1 | RouteSubjectV1; + /** For live projection this must come from validated current claim snapshots for subjectId. */ + routeSupportedClaims?: readonly string[]; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; asOf?: number; @@ -133,6 +157,14 @@ export function evaluateAllApplicableRequiredPassV1( notes: ["unsupported_verification_rule"], }; } + if (suiteManifest.evidenceLayer === "live_route_compatibility") { + if (opts.subject?.subjectKind !== "route") { + return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_subject_required"] }; + } + if (opts.routeSupportedClaims === undefined) { + return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_claim_state_required"] }; + } + } const requiredScenarios = suiteManifest.scenarios.filter( (s) => s.role === "required" || s.role === "negative_control", @@ -160,9 +192,12 @@ export function evaluateAllApplicableRequiredPassV1( } } - if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject?.subjectKind === "protocol") { if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; } + if (suiteManifest.evidenceLayer === "live_route_compatibility" && opts.subject?.subjectKind === "route") { + if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeSupportedClaims!)) continue; + } applicableRequired.push(s.id); scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } diff --git a/src/lab/subject/behavior-fingerprint.ts b/src/lab/subject/behavior-fingerprint.ts new file mode 100644 index 000000000..4cdc571aa --- /dev/null +++ b/src/lab/subject/behavior-fingerprint.ts @@ -0,0 +1,77 @@ +import { createHash } from "node:crypto"; +import { jcsStringify } from "../digest"; +import type { LabBehaviorSource, LabBehaviorValues } from "../live/types"; + +const CLOSED_KEYS = new Set([ + "wire.adapter", "wire.upstreamProtocol", "wire.responsesPath", "wire.commandCodeVersion", "wire.modelSuffixMode", + "auth.mode", "auth.transport", + "responses.stateful", "responses.upstreamStreaming", "responses.serviceTier", "responses.snapshotRepair", "responses.itemIdRepair", + "limits.contextWindow", "limits.maxInputTokens", "limits.maxOutputTokens", + "modalities.input", + "sampling.omitTemperature", "sampling.omitTopP", "sampling.omitPenalties", + "reasoning.supported", "reasoning.efforts", "reasoning.defaultEffort", "reasoning.effortMap", "reasoning.wireFormat", + "reasoning.summaryMode", "reasoning.replayMode", "reasoning.splitMode", "reasoning.toggleMode", "reasoning.budgetMode", + "tools.choiceRestrictions", "tools.parallel", "tools.hostedPreference", "tools.customFreeform", "tools.builtinNameEscaping", + "cache.forwarding", "cache.retention", "anthropic.eofPolicy", + "google.mode", "google.projectFingerprint", "google.locationFingerprint", + "openrouter.order", "openrouter.only", "openrouter.allowFallbacks", + "sidecars.vision", "sidecars.webSearch", + "mcp.maxTools", "mcp.maxSchemaBytes", "mcp.maxResultBytes", "mcp.nativeLocalExec", + "runtime.bunVersion", "runtime.platform", "runtime.arch", "runtime.streamMode", "runtime.fastMode", "runtime.effortCap", + "headers.nonCredentialBehaviorDigest", +] as const); + +const SOURCES = new Set([ + "request", "model_override", "provider_config", "registry_runtime_default", "generated_model_metadata", + "global_config", "adapter_default", "lab_forced", +]); + +const SET_ARRAY_KEYS = new Set(["modalities.input", "reasoning.efforts", "tools.choiceRestrictions", "openrouter.only"]); +const ORDERED_ARRAY_KEYS = new Set(["openrouter.order"]); +const REQUIRED_KEYS = [ + "wire.adapter", "wire.upstreamProtocol", "auth.mode", "auth.transport", "mcp.nativeLocalExec", + "runtime.bunVersion", "runtime.platform", "runtime.arch", "runtime.streamMode", "runtime.fastMode", "runtime.effortCap", +] as const; + +function canonicalArray(key: string, value: unknown[]): unknown[] { + if (SET_ARRAY_KEYS.has(key)) { + const byBytes = value.map((item) => ({ item, bytes: jcsStringify(item) })); + const seen = new Set(); + for (const row of byBytes) { + if (seen.has(row.bytes)) throw new Error(`harness_failure: duplicate behavior array value for ${key}`); + seen.add(row.bytes); + } + byBytes.sort((a, b) => a.bytes < b.bytes ? -1 : a.bytes > b.bytes ? 1 : 0); + return byBytes.map((row) => row.item); + } + if (ORDERED_ARRAY_KEYS.has(key)) return [...value]; + throw new Error(`harness_failure: unclassified_behavior_input array for ${key}`); +} + +export function normalizeBehaviorValues(values: LabBehaviorValues): LabBehaviorValues { + if (!values || typeof values !== "object" || Array.isArray(values)) { + throw new Error("harness_failure: behavior resolver output is required"); + } + for (const key of REQUIRED_KEYS) { + if (!(key in values)) throw new Error(`harness_failure: unclassified_behavior_input missing ${key}`); + } + const normalized: LabBehaviorValues = {}; + for (const key of Object.keys(values).sort()) { + if (!CLOSED_KEYS.has(key as never)) throw new Error(`harness_failure: unclassified_behavior_input ${key}`); + const row = values[key]; + if (!row || typeof row !== "object" || Array.isArray(row) || !SOURCES.has(row.source)) { + throw new Error(`harness_failure: invalid behavior source for ${key}`); + } + normalized[key] = { + source: row.source, + value: Array.isArray(row.value) ? canonicalArray(key, row.value) : row.value, + }; + } + return normalized; +} + +/** Hash the authoritative production resolver output; Lab performs validation/canonicalization only. */ +export function buildBehaviorFingerprintV1(values: LabBehaviorValues): string { + const payload = { schemaVersion: 1, resolverVersion: 1, values: normalizeBehaviorValues(values) }; + return createHash("sha256").update(jcsStringify(payload)).digest("hex"); +} diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts new file mode 100644 index 000000000..4f286ce67 --- /dev/null +++ b/src/lab/subject/installation-salt.ts @@ -0,0 +1,81 @@ +import { closeSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { dirname } from "node:path"; +import { labInstallationSaltPath, labRoot } from "../paths"; + +const SALT_BYTES = 32; +const UNSUPPORTED_DIRECTORY_FSYNC_CODES = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]); + +function readSaltFile(path: string): Uint8Array { + const bytes = readFileSync(path); + if (bytes.byteLength !== SALT_BYTES) { + throw new Error("harness_failure: invalid installation salt length"); + } + return new Uint8Array(bytes); +} + +function removeStagingFile(path: string): void { + try { unlinkSync(path); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +function fsyncDirectory(path: string): void { + // Windows has no portable directory-fsync primitive. Some POSIX-backed + // filesystems (notably virtual/shared filesystems) report directory fsync as + // unsupported. In those cases the fsync'd staging inode plus atomic same-dir + // hard-link publication is the supported durability fallback. Unexpected I/O + // errors remain fatal. + if (process.platform === "win32") return; + const dirFd = openSync(path, "r"); + try { + try { fsyncSync(dirFd); } + catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code && UNSUPPORTED_DIRECTORY_FSYNC_CODES.has(code)) return; + throw error; + } + } finally { closeSync(dirFd); } +} + +/** Read or atomically publish the per-installation salt used for local fingerprinting. */ +export function readInstallationSalt(configDir?: string): Uint8Array { + const path = labInstallationSaltPath(configDir); + const root = labRoot(configDir); + mkdirSync(root, { recursive: true, mode: 0o700 }); + + try { return readSaltFile(path); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + // The final pathname must never become visible before all 32 bytes are durable in the + // staging inode. A hard-link publish is atomic and fails with EEXIST when another process won. + const stagingPath = `${path}.tmp-${process.pid}-${randomBytes(8).toString("hex")}`; + if (dirname(stagingPath) !== dirname(path)) throw new Error("harness_failure: installation salt staging path escaped directory"); + const salt = randomBytes(SALT_BYTES); + let fd: number | undefined; + try { + fd = openSync(stagingPath, "wx", 0o600); + writeFileSync(fd, salt); + fsyncSync(fd); + closeSync(fd); + fd = undefined; + + try { + linkSync(stagingPath, path); + try { fsyncDirectory(dirname(path)); } + catch { throw new Error("harness_failure: installation salt directory fsync failed"); } + return new Uint8Array(salt); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return readSaltFile(path); + } + } finally { + if (fd !== undefined) { + try { closeSync(fd); } catch { /* best-effort close before cleanup */ } + } + removeStagingFile(stagingPath); + } +} \ No newline at end of file diff --git a/src/lab/subject/route-subject.ts b/src/lab/subject/route-subject.ts new file mode 100644 index 000000000..a5ee95ead --- /dev/null +++ b/src/lab/subject/route-subject.ts @@ -0,0 +1,73 @@ +import { localFingerprint } from "../digest"; +import type { RouteDependencyV1, RouteSubjectV1 } from "../events/types"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; +import { buildBehaviorFingerprintV1 } from "./behavior-fingerprint"; +import { readInstallationSalt } from "./installation-salt"; + +function compareDependency(a: RouteDependencyV1, b: RouteDependencyV1): number { + const fields: Array = [ + "role", "providerId", "providerInstanceFingerprint", "upstreamModelId", "endpointFingerprint", + "clientModelId", "effectiveAdapter", "upstreamProtocol", "behaviorFingerprint", + ]; + for (const field of fields) { + const av = a[field]; + const bv = b[field]; + if (av < bv) return -1; + if (av > bv) return 1; + } + return 0; +} + +function canonicalDependencies(dependencies: RouteDependencyV1[] | undefined): RouteDependencyV1[] { + const rows = (dependencies ?? []).map((row) => Object.freeze({ ...row })); + rows.sort(compareDependency); + for (let i = 1; i < rows.length; i++) { + if (compareDependency(rows[i - 1]!, rows[i]!) === 0) { + throw new Error("harness_failure: duplicate route dependency"); + } + } + return rows; +} + +/** Build RouteSubjectV1 only from an approved immutable destination and authoritative effective resolver inputs. */ +export function buildRouteSubjectV1( + routeContext: LabRouteContext, + destination: LabDestinationV1, + configDir?: string, +): RouteSubjectV1 { + if (!routeContext.providerInstanceKey) throw new Error("harness_failure: provider instance identity is required"); + if (!/^[0-9a-f]{64}$/.test(routeContext.opencodexCompatibilityVersion)) { + throw new Error("harness_failure: invalid opencodexCompatibilityVersion"); + } + const adapterValue = routeContext.behaviorValues["wire.adapter"]?.value; + const protocolValue = routeContext.behaviorValues["wire.upstreamProtocol"]?.value; + if (adapterValue !== routeContext.effectiveAdapter || protocolValue !== routeContext.upstreamProtocol) { + throw new Error("harness_failure: behavior resolver output does not match exact route"); + } + const salt = readInstallationSalt(configDir); + const providerInstanceFingerprint = localFingerprint("providerInstance", routeContext.providerInstanceKey, salt); + const dependencies = canonicalDependencies(routeContext.dependencies); + const subject: RouteSubjectV1 = { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: routeContext.providerId, + providerInstanceFingerprint, + clientModelId: routeContext.clientModelId, + upstreamModelId: routeContext.upstreamModelId, + effectiveAdapter: routeContext.effectiveAdapter, + inboundProtocol: routeContext.inboundProtocol, + upstreamProtocol: routeContext.upstreamProtocol, + surface: routeContext.surface, + opencodexCompatibilityVersion: routeContext.opencodexCompatibilityVersion, + behaviorFingerprint: buildBehaviorFingerprintV1(routeContext.behaviorValues), + endpointFingerprint: destination.fingerprint, + dependencies, + }; + return freezeRouteSubject(subject); +} + +/** Deep-freeze route dependencies as well as the subject shell. */ +export function freezeRouteSubject(subject: RouteSubjectV1): RouteSubjectV1 { + const dependencies = subject.dependencies.map((row) => Object.freeze({ ...row })); + return Object.freeze({ ...subject, dependencies: Object.freeze(dependencies) as unknown as RouteDependencyV1[] }); +} diff --git a/src/lib/lab-live-execution-authority.ts b/src/lib/lab-live-execution-authority.ts new file mode 100644 index 000000000..6bf0dbd34 --- /dev/null +++ b/src/lib/lab-live-execution-authority.ts @@ -0,0 +1,13 @@ +import type { TrustedLabRouteExecutor } from "../lab/live/types"; +import { isHostIssuedLabRouteExecutor } from "./lab-live-host"; + +/** + * Read-only trust check used by the Lab executor. + * + * Capability issuance intentionally does not live in this authority module: callers cannot + * self-attest sandbox boundary names here. Only the dedicated trusted host integration may + * construct and register an evidence-eligible route executor. + */ +export function isTrustedLabRouteExecutor(value: unknown): value is TrustedLabRouteExecutor { + return isHostIssuedLabRouteExecutor(value); +} diff --git a/src/lib/lab-live-host.ts b/src/lib/lab-live-host.ts new file mode 100644 index 000000000..9808d1745 --- /dev/null +++ b/src/lib/lab-live-host.ts @@ -0,0 +1,30 @@ +import type { LabRouteExecutor, TrustedLabRouteExecutor } from "../lab/live/types"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES } from "../lab/live/types"; + +const HOST_ISSUED_EXECUTORS = new WeakSet(); + +/** + * Trusted host integration boundary for CL-03 exact-route execution. + * + * This module is intentionally not re-exported from the Lab public surface. The host owns + * enforcement of every mandatory sandbox boundary and therefore supplies no caller-controlled + * boundary list. Adapters/providers receive only the resulting opaque capability. + * + * @internal host integration only + */ +export function createHostIssuedLabRouteExecutor(execute: LabRouteExecutor): TrustedLabRouteExecutor { + const capability: TrustedLabRouteExecutor = Object.freeze({ + execute, + enforcedBoundaries: Object.freeze([...REQUIRED_LAB_SANDBOX_BOUNDARIES]), + }); + HOST_ISSUED_EXECUTORS.add(capability); + return capability; +} + +/** Internal recognition check consumed by the read-only authority facade. */ +export function isHostIssuedLabRouteExecutor(value: unknown): value is TrustedLabRouteExecutor { + if (typeof value !== "object" || value === null || !HOST_ISSUED_EXECUTORS.has(value as object)) return false; + const capability = value as TrustedLabRouteExecutor; + return capability.enforcedBoundaries.length === REQUIRED_LAB_SANDBOX_BOUNDARIES.length + && REQUIRED_LAB_SANDBOX_BOUNDARIES.every((boundary, index) => capability.enforcedBoundaries[index] === boundary); +} diff --git a/src/lib/lab-live-pinned-sender.ts b/src/lib/lab-live-pinned-sender.ts new file mode 100644 index 000000000..d25966d3a --- /dev/null +++ b/src/lib/lab-live-pinned-sender.ts @@ -0,0 +1,45 @@ +import { PinnedHttpError, pinnedHttpGet, pinnedHttpPost } from "./pinned-http"; +import { TransportError } from "../lab/live/transport"; +import type { LabCredentialLeaseV1, LabPinnedSender } from "../lab/live/types"; + +/** Only response metadata required by current live assertions crosses into Lab. */ +const LAB_RESPONSE_HEADER_ALLOWLIST = ["content-type"] as const; + +/** + * Trusted credential/transport owner. Secret headers exist only in this non-Lab module and are + * consumed directly by the pinned HTTP primitive; they are never returned to Lab code. + */ +export function createLabAuthorizedPinnedSender( + authorize: (lease: LabCredentialLeaseV1) => Promise | HeadersInit, +): LabPinnedSender { + return async (lease, destination, pinned, request, signal, limits) => { + const headers = await authorize(lease); + const url = `${destination.scheme}://${destination.host}:${destination.port}${destination.basePath}${request.path}`; + const options = { + headers, + maxBytes: limits.maxOutputBytes, + connectTimeoutMs: limits.connectTimeoutMs, + idleTimeoutMs: Math.min(limits.firstByteTimeoutMs, limits.inactivityTimeoutMs), + rejectUnauthorized: true, + context: "Lab provider response", + }; + let response: Response; + try { + response = request.method === "POST" + ? await pinnedHttpPost(url, pinned, request.body ?? "", signal, options) + : await pinnedHttpGet(url, pinned, signal, options); + } catch (error) { + if (error instanceof PinnedHttpError && error.code === "connect_timeout") { + throw new TransportError("connect_timeout", "pinned provider connection timed out"); + } + throw error; + } + const body = await response.text(); + const responseHeaders: Record = {}; + for (const headerName of LAB_RESPONSE_HEADER_ALLOWLIST) { + const value = response.headers.get(headerName); + if (value !== null) responseHeaders[headerName] = value; + } + return { status: response.status, headers: responseHeaders, body }; + }; +} \ No newline at end of file diff --git a/src/lib/pinned-http.ts b/src/lib/pinned-http.ts index 0b7123e4a..247c94581 100644 --- a/src/lib/pinned-http.ts +++ b/src/lib/pinned-http.ts @@ -1,11 +1,20 @@ -import http, { type IncomingMessage, type RequestOptions } from "node:http"; +import http, { type ClientRequest, type IncomingMessage, type RequestOptions } from "node:http"; import https from "node:https"; export type PinnedAddress = { address: string; family: number }; +export type PinnedHttpErrorCode = "connect_timeout"; + +export class PinnedHttpError extends Error { + override readonly name = "PinnedHttpError"; + constructor(readonly code: PinnedHttpErrorCode, message: string) { super(message); } +} + export interface PinnedHttpRequestOptions { headers?: HeadersInit; maxBytes?: number; + /** Optional deadline for establishing the TCP connection and, for HTTPS, completing TLS. */ + connectTimeoutMs?: number; idleTimeoutMs?: number; rejectUnauthorized?: boolean; context?: string; @@ -27,6 +36,7 @@ function pinnedHttpRequest( throw new Error(`${options?.context ?? "request"} must use HTTP or HTTPS, got ${parsed.protocol}`); } const context = options?.context ?? "request"; + const connectTimeoutMs = options?.connectTimeoutMs; const idleTimeoutMs = options?.idleTimeoutMs ?? 60_000; const maxBytes = options?.maxBytes; const headers = new Headers(options?.headers); @@ -44,8 +54,15 @@ function pinnedHttpRequest( } let settled = false; + let req: ClientRequest | undefined; + let connectTimer: ReturnType | undefined; + const clearConnectTimer = () => { + if (connectTimer !== undefined) clearTimeout(connectTimer); + connectTimer = undefined; + }; const fail = (error: unknown) => { - try { req.destroy(); } catch { /* ignore */ } + clearConnectTimer(); + try { req?.destroy(); } catch { /* ignore */ } if (settled) return; settled = true; reject(error instanceof Error ? error : new Error(String(error))); @@ -83,6 +100,7 @@ function pinnedHttpRequest( }; const onResponse = (response: IncomingMessage) => { + clearConnectTimer(); const status = response.statusCode ?? 0; const responseHeaders = new Headers(); for (const [key, value] of Object.entries(response.headers)) { @@ -96,7 +114,7 @@ function pinnedHttpRequest( if (status < 200 || status >= 300) { try { response.destroy(); } catch { /* ignore */ } - try { req.destroy(); } catch { /* ignore */ } + try { req?.destroy(); } catch { /* ignore */ } if (settled) return; settled = true; resolve(new Response(null, { status, headers: responseHeaders })); @@ -131,7 +149,7 @@ function pinnedHttpRequest( }); }, cancel() { - req.destroy(); + req?.destroy(); }, }); @@ -141,15 +159,26 @@ function pinnedHttpRequest( }; const requestFn = parsed.protocol === "https:" ? https.request : http.request; - const req = requestFn(requestOptions, onResponse); + req = requestFn(requestOptions, onResponse); const onAbort = () => fail(signal?.reason instanceof Error ? signal.reason : new Error("aborted")); signal?.addEventListener("abort", onAbort, { once: true }); + req.on("socket", (socket) => { + if (!socket.connecting || connectTimeoutMs === undefined) return; + const connectedEvent = parsed.protocol === "https:" ? "secureConnect" : "connect"; + connectTimer = setTimeout(() => fail(new PinnedHttpError("connect_timeout", `${context} connect timed out`)), connectTimeoutMs); + socket.once(connectedEvent, clearConnectTimer); + socket.once("error", clearConnectTimer); + socket.once("close", clearConnectTimer); + }); req.setTimeout(idleTimeoutMs, () => fail(new Error(`${context} timed out`))); req.on("error", error => { signal?.removeEventListener("abort", onAbort); fail(error); }); - req.on("close", () => signal?.removeEventListener("abort", onAbort)); + req.on("close", () => { + clearConnectTimer(); + signal?.removeEventListener("abort", onAbort); + }); req.end(body); }); } @@ -179,4 +208,4 @@ export function pinnedHttpPost( options?: PinnedHttpRequestOptions, ): Promise { return pinnedHttpRequest(url, pinned, "POST", body, signal, options); -} +} \ No newline at end of file diff --git a/structure/09_compatibility-lab.md b/structure/09_compatibility-lab.md new file mode 100644 index 000000000..5535f830a --- /dev/null +++ b/structure/09_compatibility-lab.md @@ -0,0 +1,21 @@ +# Compatibility Lab SOT + +## CL-03 live-route execution boundary + +CL-03 live-route evidence is generated only for an exact `RouteSubjectV1` and remains separate from protocol-conformance and task-effectiveness evidence. + +The live runner fails closed before destination resolution unless the selected scenario is applicable and every route precondition, including explicit `lab_run_approval`, is satisfied. Destination resolution is bounded by the CL-03 connect timeout, policy-checks every resolved address, freezes the approved address set, and fingerprints only the immutable destination snapshot. Raw URLs and resolved addresses are not persisted as Lab evidence. + +Evidence-eligible route execution uses a host-issued `TrustedLabRouteExecutor`. The public Lab authority surface only recognizes host-issued capabilities; it does not expose a constructor that accepts caller-asserted sandbox boundary names. Test transports remain useful for normalization/classification tests but are never evidence-eligible. + +Successful or blocked trusted executions receive a module-private receipt bound to the canonical live authority, scenario ID, suite ID, scenario/suite manifest digests, and exact route subject ID. `observationFromLiveResult` verifies that receipt before creating directories or writing artifacts, so a structural `LiveScenarioRunResult` or mismatched case/authority cannot fabricate live evidence. + +The trusted credential sender keeps secret injection outside Lab code and uses the existing pinned HTTP primitive. CL-03 explicitly supplies its connect timeout; other pinned-HTTP callers retain their prior timeout behavior. Only response metadata required by live assertions currently crosses back into Lab (`content-type`); cookies, account/organization metadata, credential-adjacent headers, and rate-limit headers are not exposed. + +Live projection preserves the frozen `RouteSubjectV1` schema. Claim-gated scenario applicability is derived from current validated, usable `claim_snapshot` state for the exact subject rather than from caller-provided claim arrays or by extending the V1 subject preimage. A missing/wrong-kind route subject or unavailable claim state fails verification closed. + +The two machine-readable Live V1 authority copies are required to be byte-identical. Runtime loading fails closed on byte drift before parsing. Scenario limits use `perArtifactBytes` as the single per-artifact execution-limit key; the artifact policy retains its independent per-artifact policy ceiling. + +## Scope guard + +CL-03 does not expose a management CLI/API or UI. Those surfaces remain CL-04+ work. Production request routing must not synchronously trigger Compatibility Lab probing or rebuild Lab evidence. diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts new file mode 100644 index 000000000..95d40e7f0 --- /dev/null +++ b/tests/lab-live-probe.test.ts @@ -0,0 +1,185 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + CL03_LIVE_SUITES, createMockTransport, discoverLiveScenarios, evaluateAllApplicableRequiredPassV1, + executeInertTool, expandLiveScenario, invokeMcpStub, listInertToolDefinitions, listLiveScenarioIds, + loadLiveCaseAuthority, observationFromLiveResult, persistLiveResult, rebuildLabProjection, + registerMcpStubTool, routeSubjectApplicableToRequirements, runLiveScenario, runLiveSuite, + scenarioManifestDigest, subjectIdForSubject, +} from "../src/lab"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import { replayLabLedger } from "../src/lab/ledger/store"; +import { clearMcpStub } from "../src/lab/live/mcp-loopback"; +import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; +import type { LabBehaviorValues, LabRouteContext, LiveScenarioRunResult } from "../src/lab/live/types"; +import type { ProtocolSubjectV1, RouteSubjectV1 } from "../src/lab/events/types"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; + +const HOMES: string[] = []; +function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-probe-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } +afterEach(() => { for (const dir of HOMES.splice(0)) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } delete process.env.OPENCODEX_HOME; clearMcpStub(); }); + +function behavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: adapter }, "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, + "auth.mode": { source: "provider_config", value: "api_key" }, "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; +} + +function mockRoute(overrides: Partial = {}): LabRouteContext { + const base = { + providerId: "fixture-provider", providerInstanceKey: "fixture-provider-instance", clientModelId: "fixture-model", upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", + baseUrl: "https://api.example.com/v1", opencodexCompatibilityVersion: "a".repeat(64), labRunApproval: true, allowPrivateNetwork: false, + requiredClaims: ["tools", "image", "reasoning"], + availableHarnessFeatures: ["live_transport", "inert_tools", "adapter_vector", "reasoning_replay", "synthetic_image", "in_memory_mcp_stub", "mcp_call_result_v1", "mcp_lab_stub"], + }; + const merged = { ...base, ...overrides }; + return { ...merged, behaviorValues: overrides.behaviorValues ?? behavior(merged.effectiveAdapter, merged.upstreamProtocol) }; +} + +function transportForCase(caseRecord: ReturnType[number]) { + return createMockTransport({ entries: caseRecord.initiatingRequest ? [{ status: 200, headers: { "content-type": caseRecord.fixture.mediaType }, body: caseRecord.fixture.bytesUtf8 }] : [] }); +} + +function passObservation(): NormalizedObservation { + return { + client: { request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "OK" } }, + upstream: { requests: [], responses: [] }, process: { exitCode: null }, verifiers: {}, + }; +} +function reasoningObservation(): NormalizedObservation { + const observation = passObservation(); + observation.upstream.requests = [ + { status: 200, headers: {}, json: {}, rawBytes: 0 }, + { status: 200, headers: {}, json: { input: [{ signature: "sig_fixture" }] }, rawBytes: 0 }, + ]; + observation.client.response.normalizedText = ""; + return observation; +} +function trustedObservation(observation: NormalizedObservation) { + return createHostIssuedLabRouteExecutor(async () => observation); +} + +function routeSubject(): RouteSubjectV1 { + return { subjectSchemaVersion: 1, subjectKind: "route", providerId: "p", providerInstanceFingerprint: "a".repeat(64), clientModelId: "m", upstreamModelId: "m", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; +} + +describe("CL-03 live probe harness", () => { + test("loads frozen 10-case live authority with reproducible digests", () => { + const authority = loadLiveCaseAuthority(); expect(authority.cases.length).toBe(10); + expect(authority.cases.map((c) => c.id)).toContain("responses-core.live.basic-turn"); + expect(authority.cases.map((c) => c.id)).toContain("mcp-core.live.synthetic-tool"); + expect(scenarioManifestDigest(expandLiveScenario(authority.cases[0]!, authority))).toBe(scenarioManifestDigest(expandLiveScenario(authority.cases[0]!, authority))); + }); + + test("discovers all CL-03 live suites", () => { const authority = loadLiveCaseAuthority(); expect(discoverLiveScenarios(authority, CL03_LIVE_SUITES)).toHaveLength(10); expect(listLiveScenarioIds()).toHaveLength(10); }); + + test("runs only applicable live suite scenarios with an injected test transport", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); + const summary = await runLiveSuite(mockRoute(), ["responses-core", "chat-core"], { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!.fixture.bytesUtf8 }] }) }); + expect(summary.total).toBe(1); expect(summary.results[0]?.routeSubject?.subjectKind).toBe("route"); expect(summary.results[0]?.executionAuthority).toBe("test_transport"); + }); + + test("inert tool round-trips validate args and return static results", () => { expect(listInertToolDefinitions().map((t) => t.name)).toContain("lookup"); expect(executeInertTool("lookup", { q: "x" }).output).toBe("RESULT"); expect(() => executeInertTool("lookup", { q: 1 })).toThrow(); }); + test("MCP lab stub only", () => { registerMcpStubTool({ namespace: "lab", name: "echo" }); expect(invokeMcpStub("lab", "echo", { x: 1 }, { content: [{ type: "text", text: "ECHO" }] }).content[0]?.text).toBe("ECHO"); }); + + test("test transports can exercise normalization but cannot create live evidence", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: transportForCase(caseRecord) }); + expect(result.executionAuthority).toBe("test_transport"); + expect(() => observationFromLiveResult(result, caseRecord, authority, { configDir: home })).toThrow("trusted execution receipt"); + }); + + test("fabricated trusted-route result cannot create live evidence", () => { + const home = tempHome(); const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases[0]!; + const fake: LiveScenarioRunResult = { scenarioId: caseRecord.id, suite: caseRecord.suite, startedAt: 1, completedAt: 2, passed: true, classification: "inconclusive", assertionResults: [], diagnostics: [], routeSubject: routeSubject(), executionAuthority: "trusted_route" }; + expect(() => observationFromLiveResult(fake, caseRecord, authority, { configDir: home })).toThrow("trusted execution receipt"); + }); + + test("trusted receipt is bound to the executed scenario and authority", async () => { + const home = tempHome(); const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); + const differentCase = authority.cases.find((c) => c.id === "tools-core.live.function-round-trip")!; + expect(() => observationFromLiveResult(result, differentCase, authority, { configDir: home })).toThrow("receipt mismatch"); + }); + + test("unapproved routes do not resolve destinations", async () => { + const scenario = loadLiveCaseAuthority().cases[0]!; let resolves = 0; + const result = await runLiveScenario(scenario, mockRoute({ labRunApproval: false }), { resolve: async () => { resolves += 1; return [{ address: "93.184.216.34", family: 4 }]; }, transport: transportForCase(scenario) }); + expect(resolves).toBe(0); expect(result.secondaryCode).toBe("route_precondition_unmet:lab_run_approval"); expect(result.routeSubject).toBeUndefined(); + }); + + test("executor diagnostics expose only bounded codes", async () => { + const scenario = loadLiveCaseAuthority().cases[0]!; const secret = "Bearer super-secret-value"; + const result = await runLiveScenario(scenario, mockRoute(), { resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: createHostIssuedLabRouteExecutor(async () => { throw new Error(secret); }) }); + expect(result.diagnostics).toEqual(["execution_error"]); expect(JSON.stringify(result)).not.toContain(secret); + }); + + test("raw URLs and secrets never appear in persisted trusted live evidence", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(caseRecord, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); + expect(result.executionAuthority).toBe("trusted_route"); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); const serialized = JSON.stringify(event); + expect(serialized).not.toContain("api.example.com"); expect(serialized).not.toContain("https://"); expect(serialized).not.toContain("apiKey"); expect(event.evidenceLayer).toBe("live_route_compatibility"); expect(event.executionMode).toBe("live"); + }); + + test("JSONL persistence and SQLite rebuild for trusted live evidence", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((c) => c.id === "chat-core.live.basic-turn")!; + const route = mockRoute({ effectiveAdapter: "openai-chat", upstreamProtocol: "openai-chat", surface: "responses-sse" }); + const result = await runLiveScenario(caseRecord, route, { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(passObservation()) }); + persistLiveResult(result, caseRecord, authority, { configDir: home }); const projection = rebuildLabProjection(home); expect(projection.events).toBeGreaterThan(0); expect(projection.corruptions).toHaveLength(0); expect(replayLabLedger(join(home, "lab", "compatibility.jsonl")).events.length).toBeGreaterThan(0); + }); + + test("route subject applicability and cross-route evidence reuse prevention", () => { + const subjectA: RouteSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "route", providerId: "provider-a", providerInstanceFingerprint: "a".repeat(64), clientModelId: "model-a", upstreamModelId: "model-a", effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", opencodexCompatibilityVersion: "a".repeat(64), behaviorFingerprint: "b".repeat(64), endpointFingerprint: "c".repeat(64), dependencies: [] }; + const subjectB = { ...subjectA, providerId: "provider-b", endpointFingerprint: "d".repeat(64) } as RouteSubjectV1; expect(subjectIdForSubject(subjectA)).not.toBe(subjectIdForSubject(subjectB)); + const requirements = loadLiveCaseAuthority().cases[0]!.requirements; expect(routeSubjectApplicableToRequirements(requirements, subjectA, [])).toBe(true); expect(routeSubjectApplicableToRequirements({ ...requirements, inboundProtocols: ["anthropic-messages"] }, subjectA, [])).toBe(false); + }); + + test("freshness semantics for live suites", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); expect(suite.freshness.maxAgeMs).toBe(604800000); + const subject = routeSubject(); + const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject, routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + expect(result.canVerify).toBe(false); expect(result.missingRequiredScenarioIds.length).toBeGreaterThan(0); + }); + + test("live verification fails closed without route identity or validated claims", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); + const protocolSubject: ProtocolSubjectV1 = { subjectSchemaVersion: 1, subjectKind: "protocol", opencodexCompatibilityVersion: "a".repeat(64), effectiveAdapter: "openai-responses", inboundProtocol: "openai-responses", upstreamProtocol: "openai-responses", surface: "responses-http", behaviorFingerprint: "b".repeat(64) }; + const noSubject = evaluateAllApplicableRequiredPassV1(suite, [], "live", { routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + const wrongSubject = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: protocolSubject, routeSupportedClaims: [], loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + const noClaims = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: routeSubject(), loadScenarioRequirements: () => ({ inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: [], freshness: { maxAgeMs: 604800000 } }) }); + expect(noSubject.notes).toContain("route_subject_required"); expect(wrongSubject.notes).toContain("route_subject_required"); expect(noClaims.notes).toContain("route_claim_state_required"); + }); + + test("malformed requiredClaims in a live manifest is rejected", () => { + const authority = loadLiveCaseAuthority(); const suite = expandLiveSuiteManifest("responses-core", authority); + const result = evaluateAllApplicableRequiredPassV1(suite, [], "live", { subject: routeSubject(), routeSupportedClaims: [], loadScenarioManifest: () => ({ requirements: { inboundProtocols: ["openai-responses"], upstreamProtocols: ["openai-responses"], surfaces: ["responses-http"], requiredClaims: "tools" }, freshness: { maxAgeMs: 604800000 } }) }); + expect(result.canVerify).toBe(false); expect(result.notes.some((note) => note.startsWith("scenario_manifest_unavailable:"))).toBe(true); + }); + + test("timeouts classify as blockers", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const scenario = loadLiveCaseAuthority().cases[0]!; + const result = await runLiveScenario(scenario, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 200, body: "{}", error: "total_timeout" }] }) }); + expect(result.classification).toBe("timeout"); + }); + + test("HTTP auth failure is an exact-route blocker", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const scenario = loadLiveCaseAuthority().cases[0]!; + const result = await runLiveScenario(scenario, mockRoute(), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], transport: createMockTransport({ entries: [{ status: 401, body: "" }] }) }); + expect(result.classification).toBe("authentication_blocked"); expect(result.routeSubject?.endpointFingerprint).toHaveLength(64); + }); + + test("reasoning replay persists no private reasoning material", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; const authority = loadLiveCaseAuthority(); const scenario = authority.cases.find((c) => c.id === "reasoning-core.live.replay")!; + const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(reasoningObservation()) }); + expect(result.passed).toBe(true); const { event } = observationFromLiveResult(result, scenario, authority, { configDir: home }); expect(JSON.stringify(event)).not.toContain("PLAN"); + }); +}); diff --git a/tests/lab-live-receipt-integrity.test.ts b/tests/lab-live-receipt-integrity.test.ts new file mode 100644 index 000000000..5d2883e70 --- /dev/null +++ b/tests/lab-live-receipt-integrity.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadLiveCaseAuthority, observationFromLiveResult, runLiveScenario } from "../src/lab"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; +import type { LabBehaviorValues, LabRouteContext } from "../src/lab/live/types"; + +function behavior(): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: "openai-responses" }, + "wire.upstreamProtocol": { source: "lab_forced", value: "openai-responses" }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; +} + +function route(): LabRouteContext { + return { + providerId: "fixture-provider", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: [], + availableHarnessFeatures: ["live_transport"], + behaviorValues: behavior(), + }; +} + +function passingObservation(): NormalizedObservation { + return { + client: { + request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, + response: { status: 200, headers: {}, json: {}, events: [], toolCalls: [], mcpCalls: [], terminal: "completed", normalizedText: "OK" }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +test("trusted live receipt rejects post-seal outcome mutation", async () => { + const home = join(tmpdir(), `ocx-lab-receipt-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(home, { recursive: true, mode: 0o700 }); + try { + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((row) => row.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + routeExecutor: createHostIssuedLabRouteExecutor(async () => passingObservation()), + }); + expect(result.passed).toBe(true); + result.passed = false; + expect(() => observationFromLiveResult(result, scenario, authority, { configDir: home })) + .toThrow("trusted execution receipt mismatch"); + } finally { + rmSync(home, { recursive: true, force: true }); + } +}); \ No newline at end of file diff --git a/tests/lab-live-review-regressions.test.ts b/tests/lab-live-review-regressions.test.ts new file mode 100644 index 000000000..e75e52825 --- /dev/null +++ b/tests/lab-live-review-regressions.test.ts @@ -0,0 +1,224 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + buildBehaviorFingerprintV1, + buildRouteSubjectV1, + createCredentialLease, + createLabDestination, + createMockTransport, + createPinnedTransport, + expandLiveSuiteManifest, + loadLiveCaseAuthority, + runLiveScenario, + runLiveSuite, +} from "../src/lab"; +import type { + LabBehaviorValues, + LabRouteContext, + LabTransportResponse, + LiveRunConfig, +} from "../src/lab/live/types"; + +const HOMES: string[] = []; +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-live-review-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } + delete process.env.OPENCODEX_HOME; +}); + +function behavior(overrides: Partial = {}): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: "openai-responses" }, + "wire.upstreamProtocol": { source: "lab_forced", value: "openai-responses" }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + ...overrides, + }; +} + +function route(overrides: Partial = {}): LabRouteContext { + return { + providerId: "fixture-provider", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "fixture-model", + upstreamModelId: "fixture-model", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + behaviorValues: behavior(), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: ["tools", "image", "reasoning"], + availableHarnessFeatures: ["live_transport", "inert_tools", "mcp_loopback", "synthetic_image", "reasoning_replay"], + ...overrides, + }; +} + +const LIMITS: LiveRunConfig = { + totalTimeoutMs: 120_000, + connectTimeoutMs: 10_000, + firstByteTimeoutMs: 30_000, + inactivityTimeoutMs: 30_000, + maxRequests: 16, + maxInputBytes: 8 * 1024 * 1024, + maxOutputBytes: 16 * 1024 * 1024, + maxOutputTokens: 32_768, + maxToolCalls: 32, + maxMemoryBytes: 512 * 1024 * 1024, + maxChildProcesses: 0, + maxArtifacts: 16, + perArtifactBytes: 256 * 1024, + aggregateArtifactBytes: 1024 * 1024, +}; + +describe("CL-03 independent-review regressions", () => { + test("live assertions are driven by observed route bytes, not frozen fixture bytes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ entries: [{ status: 200, body: JSON.stringify({ status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "WRONG" }] }] }) }] }), + }); + expect(result.passed).toBe(false); + expect(result.classification).toBe("protocol_failure"); + }); + + test("live execution fails closed when no real route transport is supplied", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + }); + expect(result.passed).toBe(false); + expect(result.classification).toBe("harness_failure"); + expect(result.secondaryCode).toBe("live_transport_required"); + }); + + test("resolved metadata/private addresses are policy-checked, not merely recorded", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + await expect(createLabDestination({ + baseUrl: "https://safe-looking.example/v1", + labRunApproval: true, + resolve: async () => [{ address: "169.254.169.254", family: 4 }], + configDir: home, + })).rejects.toThrow(); + await expect(createLabDestination({ + baseUrl: "https://safe-looking.example/v1", + labRunApproval: true, + resolve: async () => [{ address: "10.0.0.8", family: 4 }], + configDir: home, + })).rejects.toThrow(); + }); + + test("pinned transport uses the approved address and never hostname fetch", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const lease = createCredentialLease({ destination, transportId: "test", budget: 1 }); + const seen: Array<{ address: string; host: string }> = []; + const transport = createPinnedTransport({ + destination, + lease, + transportId: "test", + limits: LIMITS, + sender: async (_lease, dest, pinned): Promise => { + seen.push({ address: pinned.address, host: dest.host }); + return { status: 200, headers: { "content-type": "application/json" }, body: "{}" }; + }, + }); + await transport.request({ method: "POST", path: "/responses", body: "{}" }); + expect(seen).toEqual([{ address: "93.184.216.34", host: "api.example.com" }]); + }); + + test("behavior fingerprint rejects unknown inputs and changes for effective behavior", () => { + const base = behavior(); + const first = buildBehaviorFingerprintV1(base); + const second = buildBehaviorFingerprintV1({ ...base, "runtime.fastMode": { source: "lab_forced", value: true } }); + expect(first).not.toBe(second); + expect(() => buildBehaviorFingerprintV1({ ...base, "secret.apiKey": { source: "provider_config", value: "nope" } } as LabBehaviorValues)).toThrow(); + }); + + test("transport blockers retain the exact already-built route subject", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const result = await runLiveScenario(scenario, route(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ entries: [{ status: 401, body: "" }] }), + }); + expect(result.classification).toBe("authentication_blocked"); + expect(result.routeSubject?.endpointFingerprint).not.toBe("0".repeat(64)); + }); + + test("runLiveSuite does not execute inapplicable route scenarios", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + let requests = 0; + const summary = await runLiveSuite(route(), ["responses-core", "chat-core"], { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + transport: createMockTransport({ + onRequest: () => { requests += 1; }, + entries: [{ status: 200, body: JSON.stringify({ status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] }) }], + }), + }); + expect(summary.total).toBe(1); + expect(requests).toBe(1); + }); + + test("codex live verification manifest includes the responses basic-turn prerequisite", () => { + const authority = loadLiveCaseAuthority(); + const suite = expandLiveSuiteManifest("codex-core", authority); + expect(suite.scenarios.map((s) => s.id)).toContain("responses-core.live.basic-turn"); + }); + + test("route subject uses configured instance identity and supplied compatibility version", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + const a = buildRouteSubjectV1(route({ providerInstanceKey: "instance-a" }), destination, home); + const b = buildRouteSubjectV1(route({ providerInstanceKey: "instance-b" }), destination, home); + expect(a.providerInstanceFingerprint).not.toBe(b.providerInstanceFingerprint); + expect(a.opencodexCompatibilityVersion).toBe("a".repeat(64)); + }); +}); diff --git a/tests/lab-live-sandbox.test.ts b/tests/lab-live-sandbox.test.ts new file mode 100644 index 000000000..26c189a54 --- /dev/null +++ b/tests/lab-live-sandbox.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + assertDestinationAddressSet, assertHostSniMatch, assertLeaseScope, buildRouteSubjectV1, classifyTransportError, + createCredentialLease, createLabDestination, createMockTransport, createSandboxResourceState, enforceSandboxLimits, + freezeRouteSubject, LabCredentialError, LabDestinationError, LabSandboxError, prepareLiveSandbox, readInstallationSalt, + rejectProxyEnvironment, TransportError, +} from "../src/lab"; +import * as executionAuthority from "../src/lib/lab-live-execution-authority"; +import { isTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import { REQUIRED_LAB_SANDBOX_BOUNDARIES, type LabBehaviorValues, type LabRouteContext, type TrustedLabRouteExecutor } from "../src/lab/live/types"; + +const HOMES: string[] = []; +function tempHome(): string { const dir = join(tmpdir(), `ocx-lab-live-${process.pid}-${Math.random().toString(16).slice(2)}`); mkdirSync(dir, { recursive: true, mode: 0o700 }); HOMES.push(dir); return dir; } +afterEach(() => { for (const dir of HOMES.splice(0)) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } } delete process.env.OPENCODEX_HOME; }); + +function behavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { "wire.adapter": { source: "lab_forced", value: adapter }, "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, "auth.mode": { source: "provider_config", value: "api_key" }, "auth.transport": { source: "provider_config", value: "authorization_bearer" }, "mcp.nativeLocalExec": { source: "lab_forced", value: false }, "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, "runtime.platform": { source: "lab_forced", value: process.platform }, "runtime.arch": { source: "lab_forced", value: process.arch }, "runtime.streamMode": { source: "lab_forced", value: "auto" }, "runtime.fastMode": { source: "lab_forced", value: false }, "runtime.effortCap": { source: "lab_forced", value: null }, "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) } }; +} +function baseRoute(overrides: Partial = {}): LabRouteContext { + const base = { providerId: "fixture-provider", providerInstanceKey: "fixture-instance", clientModelId: "fixture-model", upstreamModelId: "fixture-model", effectiveAdapter: "openai-chat", inboundProtocol: "openai-responses", upstreamProtocol: "openai-chat", surface: "responses-http", baseUrl: "https://api.example.com/v1", opencodexCompatibilityVersion: "a".repeat(64), labRunApproval: true, allowPrivateNetwork: false, requiredClaims: [] as string[], availableHarnessFeatures: ["live_transport"] }; + const merged = { ...base, ...overrides }; return { ...merged, behaviorValues: overrides.behaviorValues ?? behavior(merged.effectiveAdapter, merged.upstreamProtocol) }; +} +const limits = { totalTimeoutMs: 120000, connectTimeoutMs: 10000, firstByteTimeoutMs: 30000, inactivityTimeoutMs: 30000, maxRequests: 16, maxInputBytes: 8388608, maxOutputBytes: 16777216, maxOutputTokens: 32768, maxToolCalls: 32, maxMemoryBytes: 536870912, maxChildProcesses: 0, maxArtifacts: 16, perArtifactBytes: 262144, aggregateArtifactBytes: 1048576 }; + +function runSaltInitializer(script: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["-e", script], { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; let stderr = ""; + child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { stdout += chunk; }); + child.stderr.on("data", (chunk: string) => { stderr += chunk; }); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(stdout.trim()); + else reject(new Error(`salt initializer exited ${code}: ${stderr.trim()}`)); + }); + }); +} + +describe("CL-03 live sandbox security seams", () => { + test("rejects proxy environment variables", () => { expect(() => rejectProxyEnvironment({ HTTP_PROXY: "http://evil" })).toThrow(LabSandboxError); expect(() => rejectProxyEnvironment({ no_proxy: "localhost" })).toThrow(LabSandboxError); }); + test("prepareLiveSandbox allows only TZ=UTC and NO_COLOR=1", () => { expect(prepareLiveSandbox({})).toEqual({ TZ: "UTC", NO_COLOR: "1" }); }); + + test("DNS/address mismatch and lease scope mismatch fail closed", async () => { + const home = tempHome(); process.env.OPENCODEX_HOME = home; + const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); + expect(() => assertDestinationAddressSet(destination, [{ address: "1.2.3.4", family: 4 }])).toThrow(LabDestinationError); + const other = await createLabDestination({ baseUrl: "https://api.example.com/v2", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); + const lease = createCredentialLease({ destination, budget: 1 }); expect(() => assertLeaseScope(lease, other)).toThrow(LabCredentialError); + }); + + test("address-set equality uses the same canonical representation on both sides", async () => { + const home = tempHome(); const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 }, { address: "93.184.216.34", family: 4 }, { address: "93.184.216.34", family: 4 }], configDir: home }); + expect(() => assertDestinationAddressSet(destination, [{ address: "93.184.216.34", family: 4 }, { address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 }])).not.toThrow(); + }); + + test("destination resolution obeys the connect timeout", async () => { + const home = tempHome(); + const pending = createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, connectTimeoutMs: 5, resolve: async () => await new Promise>(() => {}), configDir: home }); + await expect(pending).rejects.toMatchObject({ code: "connect_timeout" }); + }); + + test("host/SNI mismatch fails closed", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); expect(() => assertHostSniMatch(destination, "evil.example.com")).toThrow(LabDestinationError); }); + test("metadata and link-local destinations are blocked", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://169.254.169.254/latest/meta-data", labRunApproval: true, configDir: home })).rejects.toThrow(LabDestinationError); }); + test("private network requires permission and lab approval", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; await expect(createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", configDir: home })).rejects.toThrow(LabDestinationError); const ok = await createLabDestination({ baseUrl: "http://127.0.0.1:11434/v1", allowPrivateNetwork: true, labRunApproval: true, resolve: async () => [{ address: "127.0.0.1", family: 4 }], configDir: home }); expect(ok.privateNetwork).toBe(true); }); + test("redirects fail closed in mock transport", async () => { const transport = createMockTransport({ entries: [{ status: 302, body: "", headers: { location: "https://evil.example.com" } }] }); await expect(transport.request({ method: "GET", path: "/" })).rejects.toThrow(TransportError); }); + + test("credential lease is destination-bound, bounded, and non-serializable", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const lease = createCredentialLease({ destination, budget: 1 }); assertLeaseScope(lease, destination); expect(() => JSON.stringify(lease)).toThrow(LabCredentialError); lease.consume(); expect(() => lease.consume()).toThrow(LabCredentialError); }); + test("auth/quota/network/transient classify as blockers", () => { expect(classifyTransportError(new TransportError("auth_blocked", "auth")).classification).toBe("authentication_blocked"); expect(classifyTransportError(new TransportError("quota_blocked", "quota")).classification).toBe("quota_blocked"); expect(classifyTransportError(new TransportError("network_blocked", "net")).classification).toBe("network_failure"); expect(classifyTransportError(new TransportError("provider_transient", "transient")).classification).toBe("provider_transient"); expect(classifyTransportError(new TransportError("connect_timeout", "connect")).classification).toBe("timeout"); expect(classifyTransportError(new TransportError("total_timeout", "timeout")).classification).toBe("timeout"); }); + + test("observable resource counters enforce child-process limit", () => { const state = createSandboxResourceState(); expect(() => enforceSandboxLimits(state, limits, { childProcesses: 1 })).toThrow(LabSandboxError); }); + test("untrusted runtime code cannot self-attest a trusted executor", async () => { + expect("createTrustedLabRouteExecutor" in executionAuthority).toBe(false); + const fake = { execute: async () => { throw new Error("never"); }, enforcedBoundaries: [...REQUIRED_LAB_SANDBOX_BOUNDARIES] } as TrustedLabRouteExecutor; + expect(isTrustedLabRouteExecutor(fake)).toBe(false); + const hostIssued = createHostIssuedLabRouteExecutor(async () => { throw new Error("never"); }); + expect(isTrustedLabRouteExecutor(hostIssued)).toBe(true); + expect(hostIssued.enforcedBoundaries).toEqual(REQUIRED_LAB_SANDBOX_BOUNDARIES); + }); + + test("installation salt is stable across repeated reads", () => { + const home = tempHome(); const first = readInstallationSalt(home); const second = readInstallationSalt(home); + expect(first).toEqual(second); expect(first.byteLength).toBe(32); + }); + + test("concurrent first-run salt initialization converges on one completed salt", async () => { + const home = tempHome(); + const saltModule = new URL("../src/lab/subject/installation-salt.ts", import.meta.url).href; + const script = `import { readInstallationSalt } from ${JSON.stringify(saltModule)};\nconst salt = readInstallationSalt(${JSON.stringify(home)});\nprocess.stdout.write(Buffer.from(salt).toString("hex"));`; + const [first, second] = await Promise.all([runSaltInitializer(script), runSaltInitializer(script)]); + expect(first).toHaveLength(64); expect(second).toBe(first); + expect(Buffer.from(readInstallationSalt(home)).toString("hex")).toBe(first); + }); + + test("endpoint fingerprint comes from immutable destination snapshot only", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1/custom", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subject = buildRouteSubjectV1(baseRoute({ baseUrl: "https://api.example.com/v1/custom" }), destination, home); expect(subject.endpointFingerprint).toHaveLength(64); expect(JSON.stringify(subject)).not.toContain("api.example.com"); expect(JSON.stringify(subject)).not.toContain("https://"); }); + test("route subject stability across credential rotation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const destination = await createLabDestination({ baseUrl: "https://api.example.com/v1", labRunApproval: true, resolve: async () => [{ address: "93.184.216.34", family: 4 }], configDir: home }); const subjectA = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); createCredentialLease({ destination, budget: 1 }).consume(); const subjectB = freezeRouteSubject(buildRouteSubjectV1(baseRoute(), destination, home)); expect(subjectA).toEqual(subjectB); }); +}); \ No newline at end of file