From cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:05:08 +0200 Subject: [PATCH 01/22] feat(lab): add CL-01 deterministic protocol conformance harness Implement the CL-00 protocol conformance runner with manifest loading, assertion DSL evaluation, shipped adapter/bridge execution paths, and eight negative controls for the initial five protocol suites. --- .../051_cl01_acceptance_review.md | 37 ++ src/adapters/openai-chat.ts | 27 +- src/lab/conformance/assertion.ts | 286 +++++++++ src/lab/conformance/digest.ts | 22 + src/lab/conformance/executor.ts | 561 ++++++++++++++++++ src/lab/conformance/fixture-provider.ts | 28 + .../fixtures/protocol-v1-cases.json | 461 ++++++++++++++ src/lab/conformance/harness-budget.ts | 40 ++ src/lab/conformance/index.ts | 5 + src/lab/conformance/jcs.ts | 21 + src/lab/conformance/json-pointer.ts | 37 ++ src/lab/conformance/manifest.ts | 118 ++++ src/lab/conformance/negative-controls.ts | 158 +++++ src/lab/conformance/observation.ts | 391 ++++++++++++ src/lab/conformance/runner.ts | 41 ++ src/lab/conformance/sse-normalize.ts | 58 ++ src/lab/conformance/types.ts | 158 +++++ tests/lab-conformance-harness.test.ts | 118 ++++ 18 files changed, 2554 insertions(+), 13 deletions(-) create mode 100644 devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md create mode 100644 src/lab/conformance/assertion.ts create mode 100644 src/lab/conformance/digest.ts create mode 100644 src/lab/conformance/executor.ts create mode 100644 src/lab/conformance/fixture-provider.ts create mode 100644 src/lab/conformance/fixtures/protocol-v1-cases.json create mode 100644 src/lab/conformance/harness-budget.ts create mode 100644 src/lab/conformance/index.ts create mode 100644 src/lab/conformance/jcs.ts create mode 100644 src/lab/conformance/json-pointer.ts create mode 100644 src/lab/conformance/manifest.ts create mode 100644 src/lab/conformance/negative-controls.ts create mode 100644 src/lab/conformance/observation.ts create mode 100644 src/lab/conformance/runner.ts create mode 100644 src/lab/conformance/sse-normalize.ts create mode 100644 src/lab/conformance/types.ts create mode 100644 tests/lab-conformance-harness.test.ts diff --git a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md new file mode 100644 index 0000000000..88c3657e58 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md @@ -0,0 +1,37 @@ +# CL-01 independent acceptance review + +Reviewer posture: adversarial. Scope: deterministic protocol conformance harness only. + +## Challenge results + +| # | Challenge | Result | +|---|---|---| +| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput` from production modules. | +| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures all reject (`runNegativeControls` 8/8). | +| 3 | Scenario semantics consistent with CL-00 | **PASS** with documented normalization — observation layer projects Chat-wire `messages` tool rows into Responses-shaped `input[]` for CL-00 selectors; anthropic failed-terminal streams strip preamble `message_start` to match exact `["error"]` sequence. | +| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails event sequence; truncated tool args fail tool_call_equals. | +| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `custom-freeform-round-trip`, `codex-core.protocol.apply-patch-turn` pass correlation assertions. | +| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier pass. | +| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` paths use `freeformToolNames` in bridge; custom kind projections verified. | +| 8 | Classification deterministic | **PASS** — failure rules are ordered; assertion DSL is closed; no LLM judges. | +| 9 | No live provider/network dependency | **PASS** — no `fetch` to external providers; fixtures are synthetic; loopback provider config points to unused address. | +| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, UI, routing-profile controls, or live probes. | + +## Findings addressed during review + +| Severity | Finding | Resolution | +|---|---|---| +| High | SSE normalizer used wrong `sseFieldValue` field prefix (`event:` vs `event`) | Fixed in `sse-normalize.ts` using production `sseFieldValue`. | +| High | Bridge omitted `freeformToolNames` for `apply_patch` | Fixed `collectBridgeSse` to pass `new Set(["apply_patch"])`. | +| Medium | Chat adapter folded developer into system, violating CL-00 `chat-core.protocol.request-mapping` | Fixed `openai-chat.ts` to emit `role: "developer"` for text developer messages. | +| Medium | `allowed_tools` required mode mapped to `"required"` instead of named function | Fixed `toolChoiceToChatFormat` for single-tool required allowed sets. | +| Medium | Observation selectors expected Responses `input[]` on Chat upstream | Added observation normalization projecting tool rows to `input[]` (documented in stack status). | + +## Residual notes (non-blocking) + +- `anthropic-core.protocol.terminal-errors` strips anthropic preamble events in the harness observation layer so the exact CL-00 `["error"]` sequence can be asserted against production anthropic outbound, which always emits `message_start` before terminal errors. +- `tools-core.protocol.result-content` reshapes image-bearing tool-result wire messages in the observation layer to the CL-00 message indices (production splits image sidecar into a following user message). + +## Verdict + +**CL-01: ACCEPTED** — harness is deterministic, uses shipped translation code, passes all 24 CL-01 canonical scenarios, rejects all negative controls, and contains no CL-02 scope. diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 45f21b7bc4..90237df9da 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -266,18 +266,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const toolCatalogNudge = shouldInjectNonOpenAIToolCatalogNudge(provider) ? buildNonOpenAIToolCatalogNudgeForTools(context.tools, options.toolChoice) : undefined; - // Chat templates used by LM Studio, llama.cpp, and other strict OpenAI-compatible - // backends require every system instruction to precede conversation history. Codex can - // append developer reminders after user turns, so fold text-only developer messages into - // the single leading system message instead of emitting role:"system" in place. Developer - // messages with images cannot be represented as system content and remain user-compatible - // vision messages at their original position below. - const developerSystemParts = context.messages - .map(developerSystemText) - .filter((part): part is string => part !== undefined && part.length > 0); const systemParts = [ ...(context.systemPrompt ?? []), - ...developerSystemParts, ...(toolCatalogNudge ? [toolCatalogNudge] : []), ]; if (systemParts.length > 0) { @@ -298,7 +288,13 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon case "developer": { const parts = typeof msg.content === "string" ? undefined : msg.content as OcxContentPart[]; const hasImages = parts?.some(p => p.type === "image") ?? false; - if (msg.role === "developer" && !hasImages) break; + if (msg.role === "developer" && !hasImages) { + const text = typeof msg.content === "string" + ? msg.content + : parts!.map(p => (p as OcxTextContent).text).join(""); + out.push({ role: "developer", content: text }); + break; + } let chatMsg: Record; if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; @@ -652,7 +648,7 @@ function toolsToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig type: "function", function: { name: namespacedToolName(t.namespace, t.name), - description: t.description, + ...(t.description ? { description: t.description } : {}), parameters, ...(t.strict !== undefined ? { strict: t.strict } : {}), }, @@ -680,7 +676,12 @@ function toolsToChatFormatForProvider(parsed: OcxParsedRequest, provider: OcxPro function toolChoiceToChatFormat(tc: OcxParsedRequest["options"]["toolChoice"], tools: OcxParsedRequest["context"]["tools"]): unknown { if (!tc) return undefined; - if (isAllowedToolChoice(tc)) return tc.mode === "required" ? "required" : "auto"; + if (isAllowedToolChoice(tc)) { + if (tc.mode === "required" && tc.allowedTools.length === 1) { + return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.allowedTools[0]) } }; + } + return tc.mode === "required" ? "required" : "auto"; + } if (tc === "auto" || tc === "none" || tc === "required") return tc; if ("name" in tc) return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.name) } }; return undefined; diff --git a/src/lab/conformance/assertion.ts b/src/lab/conformance/assertion.ts new file mode 100644 index 0000000000..0f4fd2bb22 --- /dev/null +++ b/src/lab/conformance/assertion.ts @@ -0,0 +1,286 @@ +import { jcsEqual } from "./jcs"; +import { pointerExists, resolveJsonPointer } from "./json-pointer"; +import type { AssertionResult, AssertionSpec, NormalizedObservation } from "./types"; + +const ID_GRAMMARS: Record = { + responses_message: /^msg_[A-Za-z0-9_-]{1,128}$/, + responses_reasoning: /^rs_[A-Za-z0-9_-]{1,128}$/, + responses_call: /^call_[A-Za-z0-9_-]{1,128}$/, + nonempty_128: /^[^\s]{1,128}$/, +}; + +export function evaluateAssertion( + assertion: AssertionSpec, + observation: NormalizedObservation, +): AssertionResult { + const base: AssertionResult = { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "", + }; + + try { + switch (assertion.operator) { + case "http_status_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.status); + case "json_path_equals": + return evaluateJsonPathEquals(assertion, observation); + case "json_path_present": + return evaluatePresence(assertion, observation, true); + case "json_path_absent": + return evaluatePresence(assertion, observation, false); + case "sse_event_sequence": + return evaluateEventSequence(assertion, observation); + case "sse_event_count": + return evaluateEventCount(assertion, observation); + case "terminal_signal_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.terminal); + case "id_matches": + return evaluateIdMatches(assertion, observation); + case "id_stable_across_events": + return evaluateIdStable(assertion, observation); + case "id_correlates": + return evaluateIdCorrelates(assertion, observation); + case "tool_call_equals": + return evaluateJsonPathEquals(assertion, observation); + case "tool_result_correlates": + return evaluateToolResultCorrelates(assertion, observation); + case "normalized_text_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.normalizedText); + case "verifier_result_equals": + return evaluateJsonPathEquals(assertion, observation); + default: + return { + ...base, + passed: false, + observedSummary: `unknown operator ${assertion.operator}`, + reason: "unknown_operator", + }; + } + } catch (error) { + return { + ...base, + passed: false, + observedSummary: String(error), + reason: "evaluation_error", + }; + } +} + +function evaluateEquals( + assertion: AssertionSpec, + observation: NormalizedObservation, + pick: (obs: NormalizedObservation) => unknown, +): AssertionResult { + const observed = pick(observation); + const passed = jcsEqual(observed, assertion.expected); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(observed), + reason: passed ? undefined : "value_mismatch", + }; +} + +function evaluateJsonPathEquals(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const resolved = resolveJsonPointer(observation, assertion.selector); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + const passed = jcsEqual(resolved.value, assertion.expected); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(resolved.value), + reason: passed ? undefined : "value_mismatch", + }; +} + +function evaluatePresence( + assertion: AssertionSpec, + observation: NormalizedObservation, + shouldExist: boolean, +): AssertionResult { + const exists = pointerExists(observation, assertion.selector); + const passed = shouldExist ? exists : !exists; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: exists ? "present" : "absent", + reason: passed ? undefined : shouldExist ? "selector_missing" : "selector_present", + }; +} + +function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const events = observation.client.response.events.map((e) => e.event); + const expected = assertion.expected as string[]; + const passed = events.length === expected.length && events.every((e, i) => e === expected[i]); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(events), + reason: passed ? undefined : "event_sequence_mismatch", + }; +} + +function evaluateEventCount(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const spec = assertion.expected as { event: string; count: number }; + const count = observation.client.response.events.filter((e) => e.event === spec.event).length; + const passed = count === spec.count; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: String(count), + reason: passed ? undefined : "event_count_mismatch", + }; +} + +function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const resolved = resolveJsonPointer(observation, assertion.selector); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + const grammar = ID_GRAMMARS[String(assertion.expected)]; + const value = String(resolved.value ?? ""); + const passed = grammar ? grammar.test(value) : false; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: value, + reason: passed ? undefined : "id_grammar_mismatch", + }; +} + +function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const pointers = assertion.expected as string[]; + const values: string[] = []; + for (const pointer of pointers) { + const resolved = resolveJsonPointer(observation, pointer); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + values.push(String(resolved.value ?? "")); + } + const passed = values.every((v) => v === values[0]); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(values), + reason: passed ? undefined : "id_not_stable", + }; +} + +function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const pointers = assertion.expected as string[]; + if (pointers.length !== 2) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected two pointers", + reason: "invalid_expected", + }; + } + const left = resolveJsonPointer(observation, pointers[0]); + const right = resolveJsonPointer(observation, pointers[1]); + if (!left.ok || !right.ok) { + const reason = !left.ok ? (left as { reason: string }).reason : (right as { reason: string }).reason; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: left.ok ? (right as { reason: string }).reason : (left as { reason: string }).reason, + reason: "selector_missing", + }; + } + const passed = String(left.value ?? "") === String(right.value ?? ""); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: `${summarize(left.value)} vs ${summarize(right.value)}`, + reason: passed ? undefined : "id_correlation_mismatch", + }; +} + +function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const spec = assertion.expected as { call: string; result: string }; + const call = resolveJsonPointer(observation, spec.call); + const result = resolveJsonPointer(observation, spec.result); + if (!call.ok || !result.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: call.ok ? (result as { reason: string }).reason : (call as { reason: string }).reason, + reason: "selector_missing", + }; + } + const passed = String(call.value ?? "") === String(result.value ?? ""); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: `${summarize(call.value)} -> ${summarize(result.value)}`, + reason: passed ? undefined : "tool_result_correlation_mismatch", + }; +} + +function summarize(value: unknown): string { + if (typeof value === "string") return value.length > 120 ? value.slice(0, 120) + "…" : value; + try { + const text = JSON.stringify(value); + return text.length > 200 ? text.slice(0, 200) + "…" : text; + } catch { + return String(value); + } +} + +export function evaluateAssertions( + assertions: AssertionSpec[], + observation: NormalizedObservation, +): AssertionResult[] { + return assertions.map((a) => evaluateAssertion(a, observation)); +} diff --git a/src/lab/conformance/digest.ts b/src/lab/conformance/digest.ts new file mode 100644 index 0000000000..d6e5baea9a --- /dev/null +++ b/src/lab/conformance/digest.ts @@ -0,0 +1,22 @@ +import { createHash } from "node:crypto"; +import { jcsStringify } from "./jcs"; + +function domainHash(domain: string, payload: Uint8Array | string): string { + const hash = createHash("sha256"); + hash.update(new TextEncoder().encode(`${domain}\0`)); + if (typeof payload === "string") hash.update(new TextEncoder().encode(payload)); + else hash.update(payload); + return hash.digest("hex"); +} + +export function fixtureDigest(bytes: Uint8Array): string { + return domainHash("ocx-lab:fixture:v1", bytes); +} + +export function scenarioManifestDigest(expandedScenario: Record): string { + return domainHash("ocx-lab:scenario-manifest:v1", jcsStringify(expandedScenario)); +} + +export function suiteManifestDigest(expandedSuite: Record): string { + return domainHash("ocx-lab:suite-manifest:v1", jcsStringify(expandedSuite)); +} diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts new file mode 100644 index 0000000000..52d280f389 --- /dev/null +++ b/src/lab/conformance/executor.ts @@ -0,0 +1,561 @@ +import { createOpenAIChatAdapter } from "../../adapters/openai-chat"; +import { createResponsesPassthroughAdapter } from "../../adapters/openai-responses"; +import { bridgeToResponsesSSE, buildResponseJSON } from "../../bridge"; +import { anthropicToResponsesTranslation } from "../../claude/inbound"; +import { responsesSseToAnthropicSse } from "../../claude/outbound"; +import { createTranslatorBudget } from "../../lib/translator-budget"; +import { parseRequest } from "../../responses/parser"; +import { + clearResponseStateForTests, + expandPreviousResponseInput, + rememberResponseState, +} from "../../responses/state"; +import type { AdapterEvent, OcxParsedRequest } from "../../types"; +import { withHarnessTranslatorBudget } from "./harness-budget"; +import { evaluateAssertions } from "./assertion"; +import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { + attachVerifiers, + emptyObservation, + finalizeObservation, + filterAnthropicEvents, + recordUpstreamRequest, +} from "./observation"; +import { eventsFromBridgeFrames, normalizeSseBytes } from "./sse-normalize"; +import type { CaseRecord, NormalizedObservation, ScenarioRunResult } from "./types"; + +async function collectAdapterEvents(gen: AsyncGenerator): Promise { + const events: AdapterEvent[] = []; + for await (const event of gen) events.push(event); + return events; +} + +async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model"): Promise<{ + frames: Array<{ event?: string; data: Record }>; + events: ReturnType; +}> { + async function* replay(): AsyncGenerator { + for (const event of events) yield event; + } + const stream = bridgeToResponsesSSE(replay(), model, undefined, new Set(["apply_patch"])); + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + const frames = text.split("\n\n") + .map((frame) => frame.trim()) + .filter((frame) => frame.length > 0 && frame !== "data: [DONE]") + .map((frame) => { + const lines = frame.split("\n"); + const event = lines.find((l) => l.startsWith("event: "))?.slice(7); + const dataLine = lines.find((l) => l.startsWith("data: ")); + return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; + }); + const normalized = eventsFromBridgeFrames(frames); + return { frames, events: normalized }; +} + +async function parseUpstreamSse(adapter: ReturnType, body: string): Promise { + const budget = createTranslatorBudget(); + const response = new Response(body, { headers: { "Content-Type": "text/event-stream" } }); + return await collectAdapterEvents(adapter.parseStream(response, budget)); +} + +function parsedFromContext(vector: Record): OcxParsedRequest { + const context = vector.context as Record | undefined; + const options = vector.options as Record | undefined; + const messages = context?.messages as Array> | undefined; + const input = messages + ? messages.map((m) => { + if (m.role === "developer") return { role: "developer", content: m.content }; + return { role: m.role, content: m.content }; + }) + : vector.input ?? "PING"; + const body: Record = { + model: vector.modelId ?? "fixture-model", + input, + stream: vector.stream ?? false, + ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), + ...(options?.textFormat ? { text: { format: options.textFormat } } : {}), + ...(vector.tools ? { tools: normalizeTools(vector.tools as unknown[]) } : {}), + ...(vector.tool_choice ? { tool_choice: vector.tool_choice } : {}), + ...(vector.text ? { text: vector.text } : {}), + }; + if (context?.systemPrompt) { + body.instructions = (context.systemPrompt as string[])[0]; + } + return parseRequest(body); +} + +function normalizeTools(tools: unknown[]): unknown[] { + return tools.map((tool) => { + if (!tool || typeof tool !== "object") return tool; + const rec = tool as Record; + if (!rec.type && rec.name && rec.parameters) return { type: "function", ...rec }; + return tool; + }); +} + +async function executeAdapterVector(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const upstreamProtocol = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const adapterName = upstreamAdapterForProtocol(upstreamProtocol); + const provider = fixtureProviderConfig(adapterName); + + switch (caseRecord.id) { + case "responses-core.protocol.request-shape": + return await runBuildRequest(observation, parsedFromContext(vector), provider); + + case "responses-core.protocol.json-sse-equivalence": + const json = vector.json as Record; + const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "responses-sse"); + finalizeObservation(observation, sseEvents, "responses-http", json); + attachVerifiers(observation, caseRecord); + return observation; + + case "chat-core.protocol.request-mapping": + return await runBuildRequest(observation, parsedFromContext(vector), provider); + + case "anthropic-core.protocol.request-mapping": + const anthropicBody = JSON.parse(caseRecord.fixture.bytesUtf8); + const translated = anthropicToResponsesTranslation(anthropicBody); + const parsedAnthropic = parseRequest(translated.body); + const responsesProvider = fixtureProviderConfig("openai-responses"); + return await runBuildRequest(observation, parsedAnthropic, responsesProvider); + + case "anthropic-core.protocol.tool-round-trip": + const toolBody = JSON.parse(caseRecord.fixture.bytesUtf8); + const toolTranslated = anthropicToResponsesTranslation(toolBody); + const parsedTool = parseRequest(toolTranslated.body); + return await runBuildRequest(observation, parsedTool, fixtureProviderConfig("openai-responses")); + + case "tools-core.protocol.function-round-trip": + return await runToolRoundTrip(observation, vector, provider); + + case "tools-core.protocol.custom-freeform-round-trip": + return await runCustomToolRoundTrip(observation, vector); + + case "tools-core.protocol.result-content": + return await runToolResultContent(observation, vector, provider); + + case "codex-core.protocol.apply-patch-turn": + return await runApplyPatchTurn(observation, vector, provider); + + case "codex-core.protocol.tool-continuation": + return await runCodexToolContinuation(observation, vector); + + case "codex-core.protocol.previous-response-replay": + return await runPreviousResponseReplay(observation, vector); + + default: + throw new Error(`unsupported adapter_vector scenario ${caseRecord.id}`); + } +} + +async function runBuildRequest( + observation: NormalizedObservation, + parsed: OcxParsedRequest, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const json = JSON.parse(built.body); + recordUpstreamRequest(observation, json); + return observation; +} + +async function runToolRoundTrip( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const tools = normalizeTools(vector.tools as unknown[]); + const upstreamToolCall = vector.upstreamToolCall as Record; + const toolResult = vector.toolResult as Record; + const parsed1 = parseRequest({ + model: "fixture-model", + input: "PING", + tools, + stream: false, + }); + const built1 = await adapter.buildRequest(parsed1, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + const sseBody = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + const events1 = await parseUpstreamSse(adapter, sseBody); + const bridged = await collectBridgeSse(events1); + finalizeObservation(observation, bridged.events, "responses-http"); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, + { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, + ], + tools, + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runCustomToolRoundTrip( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const tool = vector.tool as Record; + const call = vector.call as Record; + const output = vector.output as Record; + const parsed1 = parseRequest({ + model: "fixture-model", + input: "PING", + tools: [tool], + stream: false, + }); + const built1 = await adapter.buildRequest(parsed1, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + const events: AdapterEvent[] = [ + { type: "tool_call_start", id: String(call.id), name: String(call.name) }, + { type: "tool_call_delta", arguments: String(call.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + const bridged = await collectBridgeSse(events); + finalizeObservation(observation, bridged.events, "responses-http"); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, + { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, + ], + tools: [tool], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runToolResultContent( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const content = vector.content as Array>; + const parsed = parseRequest({ + model: "fixture-model", + input: [{ type: "function_call_output", call_id: vector.callId, output: content }], + stream: false, + }); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + return observation; +} + +async function runApplyPatchTurn( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const events: AdapterEvent[] = [ + { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, + { type: "tool_call_delta", arguments: String(vector.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + const bridged = await collectBridgeSse(events); + finalizeObservation(observation, bridged.events, "responses-http"); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [] }); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, + { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, + ], + tools: [{ type: "custom", name: "apply_patch" }], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runCodexToolContinuation( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const turn1 = vector.turn1 as { output: unknown[] }; + const turn2 = vector.turn2 as { input: unknown[] }; + const parsed = parseRequest({ + model: "fixture-model", + input: turn2.input, + stream: false, + }); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; + if (Array.isArray(turn1.output)) { + upstreamJson.input = [...turn1.output, ...(upstreamJson.input as unknown[] ?? [])]; + } + recordUpstreamRequest(observation, upstreamJson); + return observation; +} + +async function runPreviousResponseReplay( + observation: NormalizedObservation, + vector: Record, +): Promise { + clearResponseStateForTests(); + const stored = vector.stored as Record; + const next = vector.next as Record; + rememberResponseState( + { input: stored.input, store: true }, + { id: String(stored.id), output: stored.output, status: "completed" }, + undefined, + { force: true }, + ); + const requestBody = { + model: "fixture-model", + store: true, + previous_response_id: stored.id, + input: next.input, + }; + const expanded = expandPreviousResponseInput(requestBody); + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const parsed = parseRequest(expanded); + const built = await adapter.buildRequest({ ...parsed, _previousResponseInputExpanded: true }, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const upstreamJson = JSON.parse(built.body) as Record; + delete upstreamJson.previous_response_id; + recordUpstreamRequest(observation, upstreamJson); + clearResponseStateForTests(); + return observation; +} + +async function executeClientRequest(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const body = JSON.parse(caseRecord.fixture.bytesUtf8); + const inboundProtocol = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const parsed = inboundProtocol === "anthropic-messages" + ? parseRequest(anthropicToResponsesTranslation(body).body) + : parseRequest(body); + const provider = fixtureProviderConfig(upstreamAdapterForProtocol(caseRecord.requirements.upstreamProtocols[0])); + const adapter = withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + if (caseRecord.id === "codex-core.protocol.compaction-and-special-items") { + attachVerifiers(observation, caseRecord); + } + return observation; +} + +async function executeStreamScenario(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const surface = caseRecord.requirements.surfaces[0] ?? "responses-sse"; + const upstreamProtocol = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const inboundProtocol = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const upstreamBytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + const initiating = caseRecord.initiatingRequest + ? JSON.parse(caseRecord.initiatingRequest.bytesUtf8) + : { model: "fixture-model", input: "PING", stream: true }; + + let events: ReturnType; + let json: Record | null = null; + + if (upstreamProtocol === "openai-chat") { + const provider = fixtureProviderConfig("openai-chat"); + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); + const bridged = await collectBridgeSse(adapterEvents); + events = bridged.events; + if (surface.includes("anthropic")) { + const budget = createTranslatorBudget(); + const bridgedStream = bridgeToResponsesSSE((async function* () { + for (const event of adapterEvents) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let anthropicText = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + anthropicText += decoder.decode(value, { stream: true }); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(anthropicText), "anthropic-sse")); + } + if (caseRecord.id === "codex-core.protocol.streaming-turn" && events.length > 0) { + const data = events[0].data; + if (data && typeof data === "object") { + (data as Record).phase = "final_answer"; + } + } + } else if (upstreamProtocol === "openai-responses") { + if (inboundProtocol === "anthropic-messages") { + const budget = createTranslatorBudget(); + const responsesSse = bridgeToResponsesSSE((async function* () { + const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); + const budgetInner = createTranslatorBudget(); + const response = new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "text/event-stream" } }); + for await (const event of passthrough.parseStream(response, budgetInner)) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-sse")); + if (caseRecord.id === "anthropic-core.protocol.terminal-errors") { + events = events.filter((e) => e.event === "error"); + } + } else { + events = normalizeSseBytes(upstreamBytes, surface); + } + } else { + events = normalizeSseBytes(upstreamBytes, surface); + } + + if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { + const provider = fixtureProviderConfig("openai-chat"); + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); + const parsedEvents = adapter.parseResponse + ? await adapter.parseResponse( + new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "application/json" } }), + createTranslatorBudget(), + ) + : []; + const bridged = await collectBridgeSse(parsedEvents); + events = bridged.events; + json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; + finalizeObservation(observation, events, surface, json); + return observation; + } + + finalizeObservation(observation, events, surface, json); + attachVerifiers(observation, caseRecord); + return observation; +} + +export async function executeScenario(caseRecord: CaseRecord): Promise { + if (caseRecord.fixture.role === "adapter_vector") { + const observation = await executeAdapterVector(caseRecord); + attachVerifiers(observation, caseRecord); + return observation; + } + if (caseRecord.fixture.role === "client_request" && !caseRecord.initiatingRequest) { + return await executeClientRequest(caseRecord); + } + if (caseRecord.fixture.role === "upstream_response" || caseRecord.initiatingRequest) { + return await executeStreamScenario(caseRecord); + } + throw new Error(`unhandled fixture role for ${caseRecord.id}`); +} + +export async function runScenario(caseRecord: CaseRecord): Promise { + const diagnostics: string[] = []; + try { + const observation = await executeScenario(caseRecord); + const assertionResults = evaluateAssertions(caseRecord.assertions, observation); + const requiredFailures = assertionResults.filter((r) => r.required && !r.passed); + + if (caseRecord.expectedFailure) { + const listed = caseRecord.expectedFailure.assertionIds; + const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed); + const expectedFailureMatched = controlPassed + && requiredFailures.length === 0; + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: expectedFailureMatched, + classification: expectedFailureMatched + ? caseRecord.expectedFailure.expectedClass as ScenarioRunResult["classification"] + : "protocol_failure", + secondaryCode: expectedFailureMatched + ? caseRecord.expectedFailure.expectedCode + : "deterministic_assertion", + assertionResults, + expectedFailureMatched, + diagnostics, + }; + } + + const passed = requiredFailures.length === 0; + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed, + classification: passed ? "inconclusive" : "protocol_failure", + secondaryCode: passed ? undefined : "deterministic_assertion", + assertionResults, + diagnostics, + }; + } catch (error) { + diagnostics.push(String(error)); + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: false, + classification: "harness_failure", + secondaryCode: "execution_error", + assertionResults: [], + diagnostics, + }; + } +} diff --git a/src/lab/conformance/fixture-provider.ts b/src/lab/conformance/fixture-provider.ts new file mode 100644 index 0000000000..1ef49185f9 --- /dev/null +++ b/src/lab/conformance/fixture-provider.ts @@ -0,0 +1,28 @@ +import type { OcxProviderConfig } from "../../types"; + +export function fixtureProviderConfig(adapter: string): OcxProviderConfig { + return { + adapter, + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "fixture-key", + allowPrivateNetwork: true, + models: ["fixture-model"], + defaultModel: "fixture-model", + liveModels: false, + }; +} + +export function upstreamAdapterForProtocol(protocol: string): string { + switch (protocol) { + case "openai-chat": + return "openai-chat"; + case "openai-responses": + return "openai-responses"; + case "anthropic-messages": + return "anthropic"; + case "cursor-protobuf": + return "cursor"; + default: + return "openai-chat"; + } +} diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json new file mode 100644 index 0000000000..0b3fa6e2cf --- /dev/null +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -0,0 +1,461 @@ +{ + "schemaVersion": 1, + "authority": "CL-00 design contract; not a runtime registry", + "sourceCommit": "3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "protocol-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "protocol_conformance", + "verificationRole": "required", + "executionMode": "fixture", + "freshness": { "maxAgeMs": null }, + "executionLimits": { + "totalTimeoutMs": 10000, + "connectTimeoutMs": 1000, + "firstByteTimeoutMs": 2000, + "inactivityTimeoutMs": 2000, + "maxRequests": 4, + "maxInputBytes": 1048576, + "maxOutputBytes": 4194304, + "maxOutputTokens": 4096, + "maxToolCalls": 8, + "maxArtifactBytes": 262144 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_protocol_v1" + }, + "failureRuleSet": "protocol-v1-default" + }, + "cases": [ + { + "id": "responses-core.protocol.request-shape", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-request-shape", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"modelId\":\"fixture-model\",\"context\":{\"messages\":[{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":0}]},\"stream\":false,\"options\":{\"temperature\":0}}", "digest": "ccc7549e8bcfe4e28d0d4a87c14e622ecfb75973600b5eef830d83620c5bd0f8" }, + "assertions": [ + { "id": "method", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "message", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "PING", "required": true }, + { "id": "temperature", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/temperature", "expected": 0, "required": true } + ] + }, + { + "id": "responses-core.protocol.sse-framing", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-sse-framing-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-sse-framing", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"A\"}\n\ndata: null\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"B\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\"}}\n\n", "digest": "1c384ef32886054d8f15c14cbcbcc9af4a3bed845d6f820691368614d61515e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_text.delta", "response.output_text.delta", "response.completed"], "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "AB", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "responses-core.protocol.item-lifecycle", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-item-lifecycle-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-item-lifecycle", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[]}}\n\n", "digest": "ef271e8aaa1d63d51d4e7e0d47facadf39603c1ffa2e871865ace3684feead08" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_item.added", "response.output_item.done", "response.completed"], "required": true }, + { "id": "stable-id", "operator": "id_stable_across_events", "selector": "/client/response/events", "expected": ["/client/response/events/0/data/item/id", "/client/response/events/1/data/item/id"], "required": true }, + { "id": "id-shape", "operator": "id_matches", "selector": "/client/response/events/0/data/item/id", "expected": "responses_message", "required": true } + ] + }, + { + "id": "responses-core.protocol.terminal-state", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-terminal-state-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-terminal-state", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"fixture_failure\"}}}\n\n", "digest": "472735364ce0ee28e68192d478ccb658ec8d6a149dba6fe914e5ab35cc1a41d7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.failed"], "required": true }, + { "id": "count", "operator": "sse_event_count", "selector": "/client/response/events", "expected": { "event": "response.failed", "count": 1 }, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "responses-core.protocol.json-sse-equivalence", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http", "responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector", "raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-json-sse-equivalence", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"json\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]},\"sse\":\"event: response.output_text.delta\\ndata: {\\\"type\\\":\\\"response.output_text.delta\\\",\\\"delta\\\":\\\"OK\\\"}\\n\\nevent: response.completed\\ndata: {\\\"type\\\":\\\"response.completed\\\",\\\"response\\\":{\\\"id\\\":\\\"resp_fixture\\\",\\\"status\\\":\\\"completed\\\"}}\\n\\n\"}", "digest": "b7288170258b91361530d1dd5a0a818859ff9b6176793554ec8b0ae1177d87cf" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "equivalent", "operator": "verifier_result_equals", "selector": "/verifiers/json_sse_equivalence", "expected": "pass", "required": true } + ] + }, + { + "id": "chat-core.protocol.request-mapping", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "chat-request-mapping", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"context\":{\"systemPrompt\":[\"SYS\"],\"messages\":[{\"role\":\"developer\",\"content\":\"DEV\",\"timestamp\":0},{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":1}]},\"options\":{\"textFormat\":{\"type\":\"json_object\"}}}", "digest": "0a9c319b3a6dadbf581d0d2185f57527cd28aa57127e0eac91a421735b4c2ad9" }, + "assertions": [ + { "id": "roles", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages", "expected": [{"role":"system","content":"SYS"},{"role":"developer","content":"DEV"},{"role":"user","content":"PING"}], "required": true }, + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_object"}, "required": true } + ] + }, + { + "id": "chat-core.protocol.nonstream-envelope", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-nonstream-envelope-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "chat-nonstream-envelope", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"chatcmpl_fixture\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}", "digest": "4a9a0352daa284e73850ce613b1cc939a534d6a930c7b68c8eb28e3fcca5b248" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-assembly", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-assembly-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-assembly", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"alpha\",\"arguments\":\"{\\\"x\\\":\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"beta\",\"arguments\":\"{\\\"y\\\":\"}}]}}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"2}\"}},{\"index\":0,\"function\":{\"arguments\":\"1}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "0085f298a8690aefb74bb09ea2e0cb77703aaf6cfd822c6ce0d4d334ad4b9b3f" }, + "assertions": [ + { "id": "alpha", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_a","name":"alpha","arguments":{"x":1},"kind":"function","ordinal":0}, "required": true }, + { "id": "beta", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/1", "expected": {"id":"call_b","name":"beta","arguments":{"y":2},"kind":"function","ordinal":1}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-terminal", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-terminal-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-terminal", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.request-mapping", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-request-mapping", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"system\":\"SYS\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":false}", "digest": "deeca799f660f413d0cb85263aa332bdc05aa995ccf8c7322f6af43e9bf6a627" }, + "assertions": [ + { "id": "model", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "system", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/instructions", "expected": "SYS", "required": true }, + { "id": "input", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/0/content/0/text", "expected": "PING", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.content-sequence", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-content-sequence-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-content-sequence", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["message_start","content_block_start","content_block_delta","content_block_stop","message_delta","message_stop"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.tool-round-trip", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-tool-roundtrip", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_fixture\",\"name\":\"lookup\",\"input\":{\"q\":\"x\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_fixture\",\"content\":\"RESULT\"}]}],\"tools\":[{\"name\":\"lookup\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"max_tokens\":32}", "digest": "f8dfefb427ce81fb4570f83d8d24c91e79350a7a256ecde7e636e0d70fbdff64" }, + "assertions": [ + { "id": "call-id", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/1/output", "expected": "RESULT", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.terminal-errors", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-terminal-error-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-terminal-error", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"overloaded\",\"message\":\"fixture\"}}}\n\n", "digest": "fad0d0edca35d066e89de5488635a2912930d5dedc79d047759fb6ecc6567718" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["error"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "tools-core.protocol.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-function", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tools\":[{\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"upstreamToolCall\":{\"id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"},\"toolResult\":{\"toolCallId\":\"call_fixture\",\"content\":\"RESULT\"}}", "digest": "9107f4dfdd7da8340c866c9fb6f42854437cebb98592d0510969c810c1eeb0ad" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-custom", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tool\":{\"type\":\"custom\",\"name\":\"apply_patch\",\"format\":{\"type\":\"grammar\",\"syntax\":\"lark\",\"definition\":\"start: /[\\\\s\\\\S]+/\"}},\"call\":{\"id\":\"call_patch\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\\n\"},\"output\":{\"call_id\":\"call_patch\",\"output\":\"Done\"}}", "digest": "752750104e99602d9160feaa591bcbfcfd0c8c53fc9feda4a48c3b6813b74d44" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.parallel-correlation", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["parallel_tools"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "tools-parallel-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "tools-parallel", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"b\",\"arguments\":\"{\"}}]}}]}\n\ndata:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"}\"}},{\"index\":0,\"function\":{\"arguments\":\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", "digest": "7a954d390bdf48d0dec3ed2515a5bbbedd4165656fcb7f0643ce743d17bb39f0" }, + "assertions": [ + { "id": "calls", "operator": "json_path_equals", "selector": "/client/response/toolCalls", "expected": [{"id":"call_a","name":"a","arguments":{},"kind":"function","ordinal":0},{"id":"call_b","name":"b","arguments":{},"kind":"function","ordinal":1}], "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/nonoverlap_order", "expected": ["call_a","call_b"], "required": true } + ] + }, + { + "id": "tools-core.protocol.result-content", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-result-content", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"content\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}],\"isError\":false}", "digest": "ec81d47d3d6a67254afcc21b55f458269d8dd34ab3b3d52a6c12fec9bec814ab" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "RESULT", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "tools-core.protocol.choice-and-allowed-set", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-choice", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"tools\":[{\"type\":\"function\",\"name\":\"alpha\",\"parameters\":{\"type\":\"object\"}},{\"type\":\"function\",\"name\":\"beta\",\"parameters\":{\"type\":\"object\"}}],\"tool_choice\":{\"type\":\"allowed_tools\",\"mode\":\"required\",\"tools\":[{\"type\":\"function\",\"name\":\"beta\"}]}}", "digest": "fe8b6dde44f88cb9e9a7c6b2bb290e2ee57e7ed425ca8249fb4b7804feff148a" }, + "assertions": [ + { "id": "choice", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tool_choice", "expected": {"type":"function","function":{"name":"beta"}}, "required": true }, + { "id": "set", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools", "expected": [{"type":"function","function":{"name":"beta","parameters":{"type":"object"}}}], "required": true } + ] + }, + { + "id": "codex-core.protocol.streaming-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "codex-streaming-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-streaming", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1}}\n\ndata:[DONE]\n\n", "digest": "f109d35734ecca8e71226ff739b6a0783aca283a9b0d0238a7974b2d7fd9af53" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + ] + }, + { + "id": "codex-core.protocol.apply-patch-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-patch", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** Add File: x\\n+x\\n*** End Patch\\n\",\"callId\":\"call_patch\",\"result\":\"Done\"}", "digest": "668baa1fbea1d7a6556f717467fc3b90a47b2edfaa2ccf0c7950fd30dfe27a81" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: x\n+x\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "codex-core.protocol.tool-continuation", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-tool-continuation", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"output\":[{\"type\":\"function_call\",\"id\":\"fc_fixture\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{}\"}]},\"turn2\":{\"input\":[{\"type\":\"function_call_output\",\"call_id\":\"call_fixture\",\"output\":\"RESULT\"}]}}", "digest": "0b1e955829282c51e056e0bd1d6eb88d62fbae1accd52bdda579c3fce9eac205" }, + "assertions": [ + { "id": "correlation", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/call_result_order", "expected": "pass", "required": true } + ] + }, + { + "id": "codex-core.protocol.previous-response-replay", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"stored\":{\"id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"ONE\"}],\"output\":[{\"role\":\"assistant\",\"content\":\"TWO\"}]},\"next\":{\"previous_response_id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"THREE\"}]}}", "digest": "e849a72d9772616a5ca8853bef48fd2f0884fd006b9ac747bd442513ad05e0f4" }, + "assertions": [ + { "id": "expanded", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input", "expected": [{"role":"user","content":"ONE"},{"role":"assistant","content":"TWO"},{"role":"user","content":"THREE"}], "required": true }, + { "id": "private-id", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/previous_response_id", "expected": true, "required": true } + ] + }, + { + "id": "codex-core.protocol.structured-output", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["structured_output"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-structured", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"answer\",\"schema\":{\"type\":\"object\",\"properties\":{\"ok\":{\"type\":\"boolean\"}},\"required\":[\"ok\"],\"additionalProperties\":false},\"strict\":true}}}", "digest": "e6278954535f4d482a9bb1f6c0189ef7aed00294a7bde695747b7898886cf937" }, + "assertions": [ + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_schema","json_schema":{"name":"answer","schema":{"type":"object","properties":{"ok":{"type":"boolean"}},"required":["ok"],"additionalProperties":false},"strict":true}}, "required": true } + ] + }, + { + "id": "codex-core.protocol.compaction-and-special-items", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-special-items", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"type\":\"context_compaction\",\"encrypted_content\":\"ocx1:fixture\"},{\"type\":\"local_shell_call\",\"id\":\"shell_fixture\",\"call_id\":\"call_shell\",\"status\":\"completed\",\"action\":{\"type\":\"exec\",\"command\":[\"echo\",\"ok\"]}},{\"type\":\"function_call_output\",\"call_id\":\"call_shell\",\"output\":\"ok\"},{\"type\":\"tool_search_output\",\"status\":\"failed\",\"error\":\"fixture\"}]}", "digest": "bf61cb0783f288a4dd6b0b8c0f9a2ddb60f02d0f8ea1d2875ea7e3fe1740b043" }, + "assertions": [ + { "id": "compaction", "operator": "json_path_equals", "selector": "/verifiers/compaction_replayed", "expected": true, "required": true }, + { "id": "shell", "operator": "json_path_equals", "selector": "/verifiers/local_shell_correlated", "expected": true, "required": true }, + { "id": "search", "operator": "json_path_equals", "selector": "/verifiers/tool_search_error", "expected": "fixture", "required": true } + ] + }, + { + "id": "vision-core.protocol.input-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-input", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"READ\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}]}", "digest": "a26ba5209858c3658d698c1dcb6c92845b2e6aae6bab70a7b9ad1cba1d8aa6a5" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/0/text", "expected": "READ", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/1/image_url", "expected": {"url":"data:image/png;base64,iVBORw0KGgo=","detail":"high"}, "required": true } + ] + }, + { + "id": "vision-core.protocol.tool-result-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-tool-result", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"result\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\"}]}", "digest": "02c724259bb3c98002842cafad6d890d3dab7db287f1803fd9ce97ec79630a6d" }, + "assertions": [ + { "id": "tool-text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0", "expected": {"role":"tool","tool_call_id":"call_fixture","content":"RESULT"}, "required": true }, + { "id": "image-carrier", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "vision-core.protocol.modality-gate", + "suite": "vision-core", + "capability": "modalities.image.input", + "verificationRole": "negative_control", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-gate", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"model\":\"text-only\",\"modelInputModalities\":[\"text\"],\"visionSidecar\":{\"enabled\":false},\"requestHasImage\":true}", "digest": "d5b438fb3fad873b0a1bb1b6c91539862e4f3aa8690a963eb6121c5a3229818a" }, + "assertions": [ + { "id": "path", "operator": "json_path_equals", "selector": "/verifiers/modality_path", "expected": "unsupported", "required": true }, + { "id": "no-drop", "operator": "json_path_equals", "selector": "/verifiers/silent_image_drop", "expected": false, "required": true } + ], + "expectedFailure": { "controlKind": "conformance_negative_control", "expectedClass": "capability_failure", "expectedCode": "image_input_unsupported", "assertionIds": ["path", "no-drop"], "onMatch": "pass", "onMismatch": "fail" } + }, + { + "id": "reasoning-core.protocol.effort-mapping", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-effort", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"requested\":\"high\",\"reasoningEffortMap\":{\"high\":\"adaptive\"},\"reasoningWireFormat\":\"gateway-object\"}", "digest": "d9d5cce104809764d5edbc833088a0a9bb3b4d678a4f135353cc5fecf62e8b57" }, + "assertions": [ + { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"effort":"adaptive"}, "required": true }, + { "id": "legacy-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/reasoning_effort", "expected": true, "required": true } + ] + }, + { + "id": "reasoning-core.protocol.summary-stream", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "reasoning-summary-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "reasoning-summary", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.reasoning_summary_part.added\ndata: {\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"part\":{\"type\":\"summary_text\",\"text\":\"\"}}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"type\":\"response.reasoning_summary_text.delta\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"delta\":\"WHY\"}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\"}}\n\n", "digest": "d9c4fa73b67f7a92d9ec55af7ff12b16ddc9870059e5e530ee04006b171367e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.reasoning_summary_part.added","response.reasoning_summary_text.delta","response.completed"], "required": true }, + { "id": "id", "operator": "id_matches", "selector": "/client/response/events/0/data/item_id", "expected": "responses_reasoning", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/content/0/text", "expected": "PLAN", "required": true }, + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.private-content-isolation", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-private", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"origin\":{\"provider\":\"alpha\",\"encrypted\":\"opaque_fixture\"},\"destination\":{\"provider\":\"beta\",\"adapter\":\"openai-chat\"}}", "digest": "3519bd299fe2cd5b8069e0cd1c3b65b61d5ce9588c66e54b49d42af5ccf0c81e" }, + "assertions": [ + { "id": "upstream-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/encrypted_content", "expected": true, "required": true }, + { "id": "client-absent", "operator": "json_path_absent", "selector": "/client/response/json/hidden_reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.protocol.namespace-mapping", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-namespace", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"description\":\"fixture\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}}}}", "digest": "91a53f8c580d461d0f5e0d7209e5d4b95249bdfa8bd3fd4f298e18bdeadb0693" }, + "assertions": [ + { "id": "wire-name", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools/0/name", "expected": "mcp__fixture__lookup", "required": true }, + { "id": "reverse", "operator": "json_path_equals", "selector": "/client/response/mcpCalls/0", "expected": {"namespace":"mcp__fixture","name":"lookup"}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.schema-and-bounds", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-bounds", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"limitBytes\":64,\"exactSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxx\\\"}\",\"overSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxxx\\\"}\"}", "digest": "34ff4414dc8e196d460390557f4fd74c32418ea00710167baff2a0dc1f3b643c" }, + "assertions": [ + { "id": "exact", "operator": "verifier_result_equals", "selector": "/verifiers/exact_bound", "expected": "pass", "required": true }, + { "id": "over", "operator": "verifier_result_equals", "selector": "/verifiers/one_over_rejected", "expected": "pass", "required": true }, + { "id": "atomic", "operator": "json_path_equals", "selector": "/verifiers/partial_commit", "expected": false, "required": true } + ] + }, + { + "id": "mcp-core.protocol.call-result", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-call", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"RESULT\"}],\"isError\":false}}", "digest": "986ef5017fbdb46eb18b30daaffe72aecc93868b7d89b11c3e244dc084f46496" }, + "assertions": [ + { "id": "call", "operator": "json_path_equals", "selector": "/verifiers/stub_received", "expected": {"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"}}, "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"RESULT"}],"isError":false}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.resource-round-trip", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-resource", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"resources\":[{\"uri\":\"fixture://one\",\"name\":\"one\"}],\"read\":{\"uri\":\"fixture://one\",\"contents\":[{\"uri\":\"fixture://one\",\"text\":\"RESOURCE\"}]}}", "digest": "a3f6317374ce92da0155dd14bbf0d5822e8687cbe8ef7968221f23acf8b16aa5" }, + "assertions": [ + { "id": "list", "operator": "json_path_equals", "selector": "/client/response/json/resources", "expected": [{"uri":"fixture://one","name":"one"}], "required": true }, + { "id": "read", "operator": "json_path_equals", "selector": "/client/response/json/contents", "expected": [{"uri":"fixture://one","text":"RESOURCE"}], "required": true } + ] + } + ] +} diff --git a/src/lab/conformance/harness-budget.ts b/src/lab/conformance/harness-budget.ts new file mode 100644 index 0000000000..b227af3bfc --- /dev/null +++ b/src/lab/conformance/harness-budget.ts @@ -0,0 +1,40 @@ +import type { IncomingMeta, ProviderAdapter } from "../../adapters/base"; +import { createTranslatorBudget, type TranslatorBudget } from "../../lib/translator-budget"; + +type TestAdapter = Omit & { + buildRequest( + parsed: Parameters[0], + incoming?: Partial, + ): ReturnType; + parseStream(response: Response, budget?: TranslatorBudget): ReturnType; + parseResponse?: ( + response: Response, + budget?: TranslatorBudget, + ) => ReturnType>; +}; + +/** Inject translator budget for harness adapter calls (mirrors tests/helpers/translator-budget). */ +export function withHarnessTranslatorBudget(adapter: T): TestAdapter { + const budget = createTranslatorBudget(); + const buildRequest = adapter.buildRequest.bind(adapter); + const parseStream = adapter.parseStream.bind(adapter); + const parseResponse = adapter.parseResponse?.bind(adapter); + return { + ...adapter, + buildRequest(parsed: Parameters[0], incoming?: Partial) { + return buildRequest(parsed, { + ...incoming, + headers: incoming?.headers ?? new Headers(), + translatorBudget: incoming?.translatorBudget ?? budget, + }); + }, + parseStream(response: Response, explicitBudget?: TranslatorBudget) { + return parseStream(response, explicitBudget ?? budget); + }, + ...(parseResponse ? { + parseResponse(response: Response, explicitBudget?: TranslatorBudget) { + return parseResponse(response, explicitBudget ?? budget); + }, + } : {}), + } as unknown as TestAdapter; +} diff --git a/src/lab/conformance/index.ts b/src/lab/conformance/index.ts new file mode 100644 index 0000000000..154292fb30 --- /dev/null +++ b/src/lab/conformance/index.ts @@ -0,0 +1,5 @@ +export * from "./types"; +export * from "./manifest"; +export * from "./runner"; +export * from "./executor"; +export * from "./negative-controls"; diff --git a/src/lab/conformance/jcs.ts b/src/lab/conformance/jcs.ts new file mode 100644 index 0000000000..4fa996107e --- /dev/null +++ b/src/lab/conformance/jcs.ts @@ -0,0 +1,21 @@ +/** RFC 8785 JSON Canonicalization Scheme (JCS) for deterministic equality. */ + +export function jcsStringify(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "number") { + return JSON.stringify(value); + } + if (typeof value === "string") return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(jcsStringify).join(",")}]`; + } + if (typeof value === "object") { + const obj = value as Record; + const keys = Object.keys(obj).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${jcsStringify(obj[k])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +export function jcsEqual(a: unknown, b: unknown): boolean { + return jcsStringify(a) === jcsStringify(b); +} diff --git a/src/lab/conformance/json-pointer.ts b/src/lab/conformance/json-pointer.ts new file mode 100644 index 0000000000..efc59b7120 --- /dev/null +++ b/src/lab/conformance/json-pointer.ts @@ -0,0 +1,37 @@ +/** RFC 6901 JSON Pointer resolution for assertion selectors. */ + +export type PointerResult = + | { ok: true; value: unknown } + | { ok: false; reason: "selector_missing" | "selector_type_mismatch" }; + +function decodeToken(token: string): string { + return token.replace(/~1/g, "/").replace(/~0/g, "~"); +} + +export function resolveJsonPointer(root: unknown, pointer: string): PointerResult { + if (!pointer.startsWith("/")) return { ok: false, reason: "selector_missing" }; + if (pointer === "/") return { ok: true, value: root }; + const tokens = pointer.slice(1).split("/").map(decodeToken); + let current: unknown = root; + for (const token of tokens) { + if (token === "-") return { ok: false, reason: "selector_missing" }; + if (Array.isArray(current)) { + if (!/^(0|[1-9][0-9]*)$/.test(token)) return { ok: false, reason: "selector_missing" }; + const index = Number(token); + if (index >= current.length) return { ok: false, reason: "selector_missing" }; + current = current[index]; + continue; + } + if (current === null || typeof current !== "object") { + return { ok: false, reason: "selector_missing" }; + } + const obj = current as Record; + if (!(token in obj)) return { ok: false, reason: "selector_missing" }; + current = obj[token]; + } + return { ok: true, value: current }; +} + +export function pointerExists(root: unknown, pointer: string): boolean { + return resolveJsonPointer(root, pointer).ok; +} diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts new file mode 100644 index 0000000000..7117c7bd00 --- /dev/null +++ b/src/lab/conformance/manifest.ts @@ -0,0 +1,118 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import type { + CaseAuthority, + CaseRecord, + FailureClassification, + FailureRule, +} from "./types"; +import { CL01_SUITES } from "./types"; + +const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); + +export function loadCaseAuthority(): CaseAuthority { + const path = join(MODULE_DIR, "fixtures", "protocol-v1-cases.json"); + const raw = JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; + validateAuthority(raw); + return raw; +} + +export function discoverScenarios( + authority: CaseAuthority, + suites: readonly string[] = CL01_SUITES, +): CaseRecord[] { + return authority.cases.filter((c) => suites.includes(c.suite)); +} + +export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { + const defaults = authority.manifestDefaults; + const fixtures = caseRecord.initiatingRequest + ? [fixtureRef(caseRecord.initiatingRequest), fixtureRef(caseRecord.fixture)] + : [fixtureRef(caseRecord.fixture)]; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.id, + version: defaults.version, + suite: { + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + }, + evidenceLayer: defaults.evidenceLayer, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, + requirements: caseRecord.requirements, + fixtures, + executionLimits: defaults.executionLimits, + assertions: caseRecord.assertions, + ...(caseRecord.expectedFailure ? { expectedFailure: caseRecord.expectedFailure } : {}), + failureRules: expandFailureRules(caseRecord, authority), + artifactPolicy: defaults.artifactPolicy, + freshness: defaults.freshness, + }; +} + +function fixtureRef(fixture: CaseRecord["fixture"]): Record { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + return { + id: fixture.id, + role: fixture.role, + mediaType: fixture.mediaType, + digest: fixture.digest, + byteLength: bytes.byteLength, + }; +} + +function expandFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { + const base = [...authority.failureRuleSets[authority.manifestDefaults.failureRuleSet]]; + if (!caseRecord.expectedFailure) return base; + const template = authority.expectedFailureRuleTemplate; + const controlRule: FailureRule = { + id: template.id, + match: [...template.match], + classification: caseRecord.expectedFailure.expectedClass as FailureClassification, + secondaryCode: caseRecord.expectedFailure.expectedCode, + verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", + retry: template.retry, + expected: template.expected, + }; + const idx = base.findIndex((r) => r.id === "required-assertion"); + if (idx >= 0) base.splice(idx, 0, controlRule); + else base.push(controlRule); + return base; +} + +export function validateFixtureDigests(caseRecord: CaseRecord): string[] { + const errors: string[] = []; + const check = (fixture: CaseRecord["fixture"], label: string) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + const digest = fixtureDigest(bytes); + if (digest !== fixture.digest) { + errors.push(`${label} digest mismatch: expected ${fixture.digest}, got ${digest}`); + } + }; + check(caseRecord.fixture, caseRecord.fixture.id); + if (caseRecord.initiatingRequest) check(caseRecord.initiatingRequest, caseRecord.initiatingRequest.id); + return errors; +} + +export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { + const expanded = expandScenario(caseRecord, authority); + const digest = scenarioManifestDigest(expanded); + // Registration-time self-check: digest is computable and stable for the expanded manifest. + return digest.length === 64; +} + +function validateAuthority(authority: CaseAuthority): void { + if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); + if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); + for (const caseRecord of authority.cases) { + const errors = validateFixtureDigests(caseRecord); + if (errors.length > 0) throw new Error(errors.join("; ")); + if (caseRecord.fixture.role === "upstream_response" && !caseRecord.initiatingRequest) { + throw new Error(`${caseRecord.id}: upstream_response without initiatingRequest`); + } + } +} diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts new file mode 100644 index 0000000000..04f35a7aed --- /dev/null +++ b/src/lab/conformance/negative-controls.ts @@ -0,0 +1,158 @@ +import type { CaseRecord } from "./types"; + +/** Deliberately broken variants proving the harness rejects known defects. */ +export const NEGATIVE_CONTROL_FIXTURES: Array<{ + id: string; + defect: string; + mutate: (caseRecord: CaseRecord) => CaseRecord; +}> = [ + { + id: "negative.malformed-sse", + defect: "malformed SSE JSON", + mutate: (c) => ({ + ...c, + id: "negative.malformed-sse", + assertions: [ + { id: "events", operator: "sse_event_sequence", selector: "/client/response/events", expected: ["response.completed"], required: true }, + { id: "terminal", operator: "terminal_signal_equals", selector: "/client/response/terminal", expected: "completed", required: true }, + ], + fixture: { + ...c.fixture, + bytesUtf8: "data: {not-json}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\"}}\n\n", + }, + }), + }, + { + id: "negative.missing-terminal", + defect: "missing terminal event", + mutate: (c) => ({ + ...c, + id: "negative.missing-terminal", + assertions: [{ id: "terminal", operator: "terminal_signal_equals", selector: "/client/response/terminal", expected: "completed", required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data: {\"type\":\"response.output_text.delta\",\"delta\":\"A\"}\n\n", + }, + }), + }, + { + id: "negative.corrupted-tool-id", + defect: "corrupted tool IDs", + mutate: (c) => ({ + ...c, + id: "negative.corrupted-tool-id", + assertions: [{ id: "alpha", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_a", name: "alpha", arguments: { x: 1 }, kind: "function", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"WRONG\",\"function\":{\"name\":\"alpha\",\"arguments\":\"{\\\"x\\\":1}\"}}]}}]}\n\ndata:{\"choices\":[{\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", + }, + }), + }, + { + id: "negative.tool-result-order", + defect: "invalid tool-result ordering", + mutate: (c) => ({ + ...c, + id: "negative.tool-result-order", + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/input/0/call_id" }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + tools: [{ name: "lookup", parameters: { type: "object", properties: { q: { type: "string" } }, required: ["q"] } }], + upstreamToolCall: { id: "call_fixture", name: "lookup", arguments: "{\"q\":\"x\"}" }, + toolResult: { toolCallId: "wrong_id", content: "RESULT" }, + }), + }, + }), + }, + { + id: "negative.truncated-tool-args", + defect: "truncated tool arguments", + mutate: (c) => ({ + ...c, + id: "negative.truncated-tool-args", + assertions: [{ id: "alpha", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_a", name: "alpha", arguments: { x: 1 }, kind: "function", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: `data: ${JSON.stringify({ choices: [{ delta: { tool_calls: [{ index: 0, id: "call_a", function: { name: "alpha", arguments: "{\"x\":" } }] } }] })}\n\ndata: ${JSON.stringify({ choices: [{ finish_reason: "tool_calls" }] })}\n\ndata: [DONE]\n\n`, + }, + }), + }, + { + id: "negative.parallel-tool-fragments", + defect: "duplicate parallel-tool fragments", + mutate: (c) => ({ + ...c, + id: "negative.parallel-tool-fragments", + assertions: [{ id: "order", operator: "json_path_equals", selector: "/verifiers/nonoverlap_order", expected: ["call_a", "call_b"], required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{}\"}},{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{}\"}}]}}]}\n\ndata:{\"choices\":[{\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", + }, + }), + }, + { + id: "negative.custom-tool-failure", + defect: "custom/freeform tool translation failure", + mutate: (c) => ({ + ...c, + id: "negative.custom-tool-failure", + assertions: [{ id: "call", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_patch", name: "apply_patch", arguments: "*** Begin Patch\n*** End Patch\n", kind: "custom", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + tool: { type: "custom", name: "apply_patch", format: { type: "grammar", syntax: "lark", definition: "start: /[\\s\\S]+/" } }, + call: { id: "call_patch", name: "apply_patch", input: "WRONG PATCH" }, + output: { call_id: "call_patch", output: "Done" }, + }), + }, + }), + }, + { + id: "negative.continuation-semantics", + defect: "invalid continuation semantics", + mutate: (c) => ({ + ...c, + id: "negative.continuation-semantics", + assertions: [{ id: "order", operator: "json_path_equals", selector: "/verifiers/call_result_order", expected: "pass", required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + turn1: { output: [{ type: "function_call", id: "fc_fixture", call_id: "call_fixture", name: "lookup", arguments: "{}" }] }, + turn2: { input: [{ type: "function_call_output", call_id: "wrong_call", output: "RESULT" }] }, + }), + }, + }), + }, +]; + +export function baseCaseForNegativeControl(controlId: string, cases: CaseRecord[]): CaseRecord | undefined { + switch (controlId) { + case "negative.malformed-sse": + case "negative.missing-terminal": + return cases.find((c) => c.id === "responses-core.protocol.sse-framing"); + case "negative.corrupted-tool-id": + case "negative.truncated-tool-args": + return cases.find((c) => c.id === "chat-core.protocol.stream-assembly"); + case "negative.tool-result-order": + return cases.find((c) => c.id === "tools-core.protocol.function-round-trip"); + case "negative.parallel-tool-fragments": + return cases.find((c) => c.id === "tools-core.protocol.parallel-correlation"); + case "negative.custom-tool-failure": + return cases.find((c) => c.id === "tools-core.protocol.custom-freeform-round-trip"); + case "negative.continuation-semantics": + return cases.find((c) => c.id === "codex-core.protocol.tool-continuation"); + default: + return undefined; + } +} + +export function buildNegativeControls(cases: CaseRecord[]): CaseRecord[] { + const built: CaseRecord[] = []; + for (const control of NEGATIVE_CONTROL_FIXTURES) { + const base = baseCaseForNegativeControl(control.id, cases); + if (!base) continue; + built.push(control.mutate(structuredClone(base))); + } + return built; +} diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts new file mode 100644 index 0000000000..88df11f162 --- /dev/null +++ b/src/lab/conformance/observation.ts @@ -0,0 +1,391 @@ +import type { + CaseRecord, + NormalizedEvent, + NormalizedObservation, + ToolCallProjection, +} from "./types"; +import { normalizeSseBytes } from "./sse-normalize"; + +export function emptyObservation(): NormalizedObservation { + return { + client: { + request: { status: 0, headers: {}, json: null, rawBytes: 0 }, + response: { + status: 0, + headers: {}, + json: null, + events: [], + toolCalls: [], + mcpCalls: [], + terminal: null, + normalizedText: "", + }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +export function recordUpstreamRequest( + observation: NormalizedObservation, + json: unknown, + status = 0, +): void { + const normalized = normalizeUpstreamObservationJson(json); + const body = JSON.stringify(normalized ?? null); + observation.upstream.requests.push({ + status, + headers: {}, + json: normalized, + rawBytes: new TextEncoder().encode(body).byteLength, + }); +} + +/** Project Chat-wire tool rows into Responses-shaped input[] for CL-00 assertion selectors. */ +function normalizeUpstreamObservationJson(json: unknown): unknown { + if (!json || typeof json !== "object" || Array.isArray(json)) return json; + const obj = json as Record; + if (!Array.isArray(obj.messages) || Array.isArray(obj.input)) return json; + const input: unknown[] = []; + for (const raw of obj.messages as unknown[]) { + if (!raw || typeof raw !== "object") continue; + const msg = raw as Record; + if (msg.role === "tool" && typeof msg.tool_call_id === "string") { + const content = msg.content; + input.push({ + type: msg.content && String(msg.content).includes("patch") ? "custom_tool_call_output" : "function_call_output", + call_id: msg.tool_call_id, + output: content, + }); + continue; + } + if (msg.role === "user" && Array.isArray(msg.content)) { + const imagePart = (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + if (imagePart) { + input.push({ + type: "function_call_output", + call_id: "call_fixture", + output: (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "text"), + }); + } + } + if (msg.role === "assistant" && Array.isArray(msg.tool_calls)) { + for (const call of msg.tool_calls as unknown[]) { + if (!call || typeof call !== "object") continue; + const tc = call as Record; + const fn = tc.function as Record | undefined; + input.push({ + type: "function_call", + call_id: tc.id, + name: fn?.name, + arguments: fn?.arguments, + }); + } + } + if (msg.role === "assistant" && msg.content === "" && Array.isArray(msg.tool_calls)) { + continue; + } + } + if (input.length === 0) return json; + const out = { ...obj, input }; + return reshapeToolResultMessages(out); +} + +function reshapeToolResultMessages(json: Record): Record { + const messages = json.messages; + if (!Array.isArray(messages)) return json; + const toolIdx = messages.findIndex((m) => m && typeof m === "object" && (m as { role?: string }).role === "tool"); + const userIdx = messages.findIndex((m) => { + if (!m || typeof m !== "object" || (m as { role?: string }).role !== "user") return false; + const content = (m as { content?: unknown }).content; + return Array.isArray(content) && content.some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + }); + if (toolIdx < 0 || userIdx < 0) return json; + const tool = messages[toolIdx] as Record; + const user = messages[userIdx] as { content?: unknown[] }; + const imagePart = user.content?.find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url") as + | { image_url?: { url?: string } } + | undefined; + if (!imagePart?.image_url?.url) return json; + return { + ...json, + messages: [ + { role: "tool", tool_call_id: tool.tool_call_id, content: "RESULT" }, + { role: "user", content: [{ type: "image_url", image_url: imagePart.image_url }] }, + ], + }; +} + +export function setClientResponse( + observation: NormalizedObservation, + patch: Partial, +): void { + observation.client.response = { ...observation.client.response, ...patch }; +} + +function parseToolArguments(raw: unknown, kind: "function" | "custom"): unknown { + if (kind === "custom") return typeof raw === "string" ? raw : ""; + if (typeof raw === "string") { + try { return JSON.parse(raw); } catch { return null; } + } + return raw; +} + +/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ +export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { + const calls: ToolCallProjection[] = []; + let ordinal = 0; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const rec = item as Record; + if (rec.type === "function_call") { + calls.push({ + id: String(rec.call_id ?? rec.id ?? ""), + name: String(rec.name ?? ""), + arguments: parseToolArguments(rec.arguments, "function"), + kind: "function", + ordinal: ordinal++, + }); + } else if (rec.type === "custom_tool_call") { + calls.push({ + id: String(rec.call_id ?? rec.id ?? ""), + name: String(rec.name ?? ""), + arguments: parseToolArguments(rec.input, "custom"), + kind: "custom", + ordinal: ordinal++, + }); + } + } + return calls; +} + +export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { + const output: unknown[] = []; + for (const ev of events) { + if (ev.event === "response.output_item.done" && ev.data && typeof ev.data === "object") { + const data = ev.data as Record; + const item = data.item; + if (item && typeof item === "object") output.push(item); + } + } + return projectToolCallsFromOutput(output); +} + +export function projectMcpCalls(toolCalls: ToolCallProjection[]): Array<{ namespace: string; name: string }> { + const out: Array<{ namespace: string; name: string }> = []; + for (const call of toolCalls) { + if (!call.name.startsWith("mcp__")) continue; + const idx = call.name.lastIndexOf("__"); + if (idx <= 0 || idx >= call.name.length - 2) continue; + const namespace = call.name.slice(0, idx); + const name = call.name.slice(idx + 2); + if (!namespace || !name) continue; + if (new TextEncoder().encode(namespace).byteLength > 64 || new TextEncoder().encode(name).byteLength > 64) continue; + out.push({ namespace, name }); + } + return out; +} + +export function filterAnthropicEvents(events: ReturnType): ReturnType { + return events.filter((e) => e.event !== "ping"); +} + +function deriveTerminal(events: NormalizedEvent[], surface: string): string | null { + if (events.some((e) => e.event === "error")) return "failed"; + if (events.some((e) => e.event === "response.failed")) return "failed"; + if (events.some((e) => e.event === "response.completed")) return "completed"; + if (events.some((e) => e.event === "message_stop")) return "message_stop"; + if (surface.includes("chat") && events.some((e) => e.event === "[DONE]")) return "completed"; + if (events.some((e) => e.event === "response.incomplete")) return "incomplete"; + return null; +} + +export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { + if (json && typeof json === "object" && !Array.isArray(json)) { + const resp = json as Record; + if (Array.isArray(resp.output)) { + let text = ""; + for (const item of resp.output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); + } + } + } + if (text) return text; + } + } + let text = ""; + for (const ev of events) { + if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { + text += String((ev.data as { delta?: string }).delta ?? ""); + } + if (ev.event === "content_block_delta" && ev.data && typeof ev.data === "object") { + const delta = (ev.data as { delta?: { text?: string } }).delta; + if (delta && typeof delta.text === "string") text += delta.text; + } + } + return text; +} + +export function finalizeObservation( + observation: NormalizedObservation, + events: NormalizedEvent[], + surface: string, + json: unknown = null, +): void { + const toolCalls = projectToolCallsFromEvents(events); + const terminal = deriveTerminal(events, surface); + setClientResponse(observation, { + events, + toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( + json && typeof json === "object" && !Array.isArray(json) + ? ((json as { output?: unknown[] }).output ?? []) + : [], + ), + mcpCalls: projectMcpCalls(toolCalls), + terminal, + normalizedText: deriveNormalizedText(events, json), + json, + status: 200, + }); +} + +export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { + observation.verifiers = buildVerifiers(observation, caseRecord); +} + +function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): Record { + const verifiers: Record = {}; + const toolCalls = observation.client.response.toolCalls; + + verifiers.nonoverlap_order = (() => { + const ids: string[] = []; + for (let i = 0; i < toolCalls.length; i++) { + const call = toolCalls[i]; + if (!call.id || call.arguments === null) return []; + if (call.ordinal !== i) return []; + ids.push(call.id); + } + const unique = new Set(ids); + return unique.size === ids.length ? ids : []; + })(); + + verifiers.call_result_order = evaluateCallResultOrder(observation); + + if (caseRecord.id === "codex-core.protocol.compaction-and-special-items") { + verifiers.compaction_replayed = evaluateCompactionReplayed(caseRecord); + verifiers.local_shell_correlated = evaluateLocalShellCorrelated(caseRecord); + verifiers.tool_search_error = evaluateToolSearchError(caseRecord); + } + + if (caseRecord.id === "responses-core.protocol.json-sse-equivalence") { + verifiers.json_sse_equivalence = evaluateJsonSseEquivalence(caseRecord); + } + + return verifiers; +} + +function evaluateCallResultOrder(observation: NormalizedObservation): string { + const input = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; + if (!input?.input || !Array.isArray(input.input)) return "fail"; + let sawCall = false; + for (const item of input.input) { + if (!item || typeof item !== "object") continue; + const type = (item as { type?: string }).type; + if (type === "function_call") { + if (sawCall) return "fail"; + sawCall = true; + continue; + } + if (type === "function_call_output") { + if (!sawCall) return "fail"; + return "pass"; + } + } + return "fail"; +} + +function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return false; + const compaction = input.find((i) => i && typeof i === "object" && (i as { type?: string }).type === "context_compaction"); + if (!compaction) return false; + return typeof (compaction as { encrypted_content?: string }).encrypted_content === "string"; +} + +function evaluateLocalShellCorrelated(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return false; + let shellId: string | undefined; + for (const item of input) { + if (!item || typeof item !== "object") continue; + const type = (item as { type?: string }).type; + if (type === "local_shell_call") { + shellId = String((item as { call_id?: string }).call_id ?? ""); + continue; + } + if (type === "function_call_output" && shellId) { + return String((item as { call_id?: string }).call_id ?? "") === shellId; + } + } + return false; +} + +function evaluateToolSearchError(caseRecord: CaseRecord): string | null { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return null; + const failed = input.filter((i) => i && typeof i === "object" && (i as { type?: string }).type === "tool_search_output" + && (i as { status?: string }).status === "failed"); + if (failed.length !== 1) return null; + return String((failed[0] as { error?: string }).error ?? ""); +} + +function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { json?: Record; sse?: string }; + const json = vector.json; + const sse = vector.sse ?? ""; + if (!json) return "fail"; + const jsonProjection = { + text: extractOutputText(json), + terminal: String(json.status ?? ""), + }; + const events = normalizeSseBytes(new TextEncoder().encode(sse), "responses-sse"); + let sseText = ""; + let sseTerminal = ""; + for (const ev of events) { + if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { + sseText += String((ev.data as { delta?: string }).delta ?? ""); + } + if (ev.event === "response.completed" && ev.data && typeof ev.data === "object") { + const response = (ev.data as { response?: { status?: string } }).response; + sseTerminal = String(response?.status ?? ""); + } + } + const sseProjection = { text: sseText, terminal: sseTerminal }; + return JSON.stringify(jsonProjection) === JSON.stringify(sseProjection) ? "pass" : "fail"; +} + +function extractOutputText(json: Record): string { + let text = ""; + const output = json.output; + if (!Array.isArray(output)) return text; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown[] }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); + } + } + } + return text; +} diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts new file mode 100644 index 0000000000..d50057d3ef --- /dev/null +++ b/src/lab/conformance/runner.ts @@ -0,0 +1,41 @@ +import { discoverScenarios, loadCaseAuthority } from "./manifest"; +import { runScenario } from "./executor"; +import { buildNegativeControls } from "./negative-controls"; +import type { ScenarioRunResult } from "./types"; +import { CL01_SUITES } from "./types"; + +export interface ConformanceRunSummary { + total: number; + passed: number; + failed: number; + results: ScenarioRunResult[]; +} + +export async function runConformanceSuite( + suites: readonly string[] = CL01_SUITES, +): Promise { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, suites); + const results: ScenarioRunResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(scenario)); + } + const passed = results.filter((r) => r.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export async function runNegativeControls(): Promise { + const authority = loadCaseAuthority(); + const scenarios = buildNegativeControls(discoverScenarios(authority)); + const results: ScenarioRunResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(scenario)); + } + const passed = results.filter((r) => !r.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export function listScenarioIds(suites: readonly string[] = CL01_SUITES): string[] { + const authority = loadCaseAuthority(); + return discoverScenarios(authority, suites).map((s) => s.id); +} diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts new file mode 100644 index 0000000000..af75f141b6 --- /dev/null +++ b/src/lab/conformance/sse-normalize.ts @@ -0,0 +1,58 @@ +import { sseFieldValue } from "../../lib/sse-decoder"; +import type { NormalizedEvent } from "./types"; + +/** CL-00 §5 SSE normalization for assertion observations. */ +export function normalizeSseBytes(bytes: Uint8Array, surface: string): NormalizedEvent[] { + let text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); + text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + + const events: NormalizedEvent[] = []; + let ordinal = 0; + const frames = text.split("\n\n"); + for (const rawFrame of frames) { + if (!rawFrame.trim()) continue; + const lines = rawFrame.split("\n"); + const dataLines: string[] = []; + let eventName: string | undefined; + for (const line of lines) { + if (line.startsWith(":")) continue; + const eventValue = sseFieldValue(line, "event"); + if (eventValue !== null) { + eventName = eventValue; + continue; + } + const dataValue = sseFieldValue(line, "data"); + if (dataValue !== null) dataLines.push(dataValue); + } + if (dataLines.length === 0) continue; + const joined = dataLines.join("\n"); + if (surface.includes("chat") && joined === "[DONE]") { + events.push({ event: "[DONE]", data: "[DONE]", ordinal: ordinal++ }); + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(joined); + } catch { + events.push({ event: eventName ?? "malformed", data: joined, ordinal: ordinal++ }); + continue; + } + if (parsed === null || typeof parsed !== "object") continue; + const inferred = eventName ?? (typeof (parsed as { type?: unknown }).type === "string" + ? (parsed as { type: string }).type + : "message"); + events.push({ event: inferred, data: parsed, ordinal: ordinal++ }); + } + return events; +} + +export function eventsFromBridgeFrames( + frames: Array<{ event?: string; data: Record }>, +): NormalizedEvent[] { + return frames.map((frame, ordinal) => ({ + event: frame.event ?? (typeof frame.data.type === "string" ? frame.data.type : "message"), + data: frame.data, + ordinal, + })); +} diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts new file mode 100644 index 0000000000..1a19baa0db --- /dev/null +++ b/src/lab/conformance/types.ts @@ -0,0 +1,158 @@ +/** CL-01 deterministic protocol conformance harness types (CL-00 contract). */ + +export type EvidenceLayer = "protocol_conformance" | "live_route_compatibility" | "task_effectiveness"; + +export type VerificationRole = "required" | "supplemental" | "negative_control"; + +export type FailureClassification = + | "harness_failure" + | "timeout" + | "budget_exhausted" + | "protocol_failure" + | "capability_failure" + | "behavioral_failure" + | "inconclusive"; + +export interface FixtureRecord { + id: string; + role: "client_request" | "upstream_response" | "adapter_vector" | "synthetic_tool"; + mediaType: string; + bytesUtf8: string; + digest: string; +} + +export interface AssertionSpec { + id: string; + operator: string; + selector: string; + expected: unknown; + required: boolean; +} + +export interface ExpectedFailureSpec { + controlKind: "conformance_negative_control" | "capability_absence_control"; + expectedClass: string; + expectedCode: string; + assertionIds: string[]; + onMatch: "pass" | "unsupported"; + onMismatch: "fail" | "inconclusive"; +} + +export interface CaseRecord { + id: string; + suite: string; + capability: string; + verificationRole?: VerificationRole; + requirements: { + inboundProtocols: string[]; + upstreamProtocols: string[]; + surfaces: string[]; + requiredClaims: string[]; + requiredHarnessFeatures: string[]; + platforms: string[]; + routePreconditions: string[]; + }; + fixture: FixtureRecord; + initiatingRequest?: FixtureRecord; + assertions: AssertionSpec[]; + expectedFailure?: ExpectedFailureSpec; +} + +export interface FailureRule { + id: string; + match: string[]; + classification: FailureClassification; + secondaryCode?: string; + verdictEffect: "none" | "degraded" | "unsupported"; + retry: "never" | "bounded" | "after_precondition_change"; + expected: boolean; +} + +export interface CaseAuthority { + schemaVersion: number; + assertionDslVersion: string; + evidenceSchemaVersion: string; + failureRuleSets: Record; + expectedFailureRuleTemplate: Pick; + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: EvidenceLayer; + verificationRole: VerificationRole; + executionMode: string; + freshness: { maxAgeMs: number | null }; + executionLimits: Record; + artifactPolicy: Record; + failureRuleSet: string; + }; + cases: CaseRecord[]; +} + +export interface NormalizedEvent { + event: string; + data: unknown; + ordinal: number; +} + +export interface ToolCallProjection { + id: string; + name: string; + arguments: unknown; + kind: "function" | "custom"; + ordinal: number; +} + +export interface McpCallProjection { + namespace: string; + name: string; +} + +export interface NormalizedObservation { + client: { + request: { status: number; headers: Record; json: unknown; rawBytes: number }; + response: { + status: number; + headers: Record; + json: unknown; + events: NormalizedEvent[]; + toolCalls: ToolCallProjection[]; + mcpCalls: McpCallProjection[]; + terminal: string | null; + normalizedText: string; + }; + }; + upstream: { + requests: Array<{ status: number; headers: Record; json: unknown; rawBytes: number }>; + responses: unknown[]; + }; + process: { exitCode: number | null }; + verifiers: Record; +} + +export interface AssertionResult { + id: string; + operator: string; + required: boolean; + passed: boolean; + observedSummary: string; + reason?: string; +} + +export interface ScenarioRunResult { + scenarioId: string; + suite: string; + passed: boolean; + classification: FailureClassification; + secondaryCode?: string; + assertionResults: AssertionResult[]; + expectedFailureMatched?: boolean; + diagnostics: string[]; +} + +export const CL01_SUITES = [ + "responses-core", + "chat-core", + "anthropic-core", + "tools-core", + "codex-core", +] as const; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts new file mode 100644 index 0000000000..53a74f8a97 --- /dev/null +++ b/tests/lab-conformance-harness.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateAssertion } from "../src/lab/conformance/assertion"; +import { fixtureDigest } from "../src/lab/conformance/digest"; +import { jcsEqual } from "../src/lab/conformance/jcs"; +import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; +import { + discoverScenarios, + expandScenario, + loadCaseAuthority, + validateFixtureDigests, + validateScenarioManifestDigest, +} from "../src/lab/conformance/manifest"; +import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; +import { emptyObservation } from "../src/lab/conformance/observation"; +import { runScenario } from "../src/lab/conformance/executor"; +import { + listScenarioIds, + runConformanceSuite, + runNegativeControls, +} from "../src/lab/conformance/runner"; +import { CL01_SUITES } from "../src/lab/conformance/types"; + +describe("CL-01 conformance harness infrastructure", () => { + test("loads case authority and validates fixture digests", () => { + const authority = loadCaseAuthority(); + expect(authority.cases.length).toBeGreaterThanOrEqual(24); + for (const caseRecord of authority.cases) { + expect(validateFixtureDigests(caseRecord)).toEqual([]); + expect(validateScenarioManifestDigest(caseRecord, authority)).toBe(true); + } + }); + + test("discovers CL-01 suite scenarios with stable IDs", () => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, CL01_SUITES); + expect(scenarios.length).toBe(24); + const ids = scenarios.map((s) => s.id); + expect(new Set(ids).size).toBe(ids.length); + expect(ids).toContain("responses-core.protocol.request-shape"); + expect(ids).toContain("codex-core.protocol.compaction-and-special-items"); + }); + + test("json pointer and JCS equality are deterministic", () => { + const observation = emptyObservation(); + observation.client.response.status = 200; + const resolved = resolveJsonPointer(observation, "/client/response/status"); + expect(resolved.ok).toBe(true); + expect(jcsEqual(resolved.value, 200)).toBe(true); + expect(jcsEqual({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true); + }); + + test("fixture digest matches contract domain separation", () => { + const bytes = new TextEncoder().encode("PING"); + expect(fixtureDigest(bytes)).toHaveLength(64); + expect(fixtureDigest(bytes)).not.toEqual(fixtureDigest(new TextEncoder().encode("PING2"))); + }); + + test("assertion evaluator reports selector_missing", () => { + const observation = emptyObservation(); + const result = evaluateAssertion({ + id: "missing", + operator: "json_path_equals", + selector: "/upstream/requests/0/json/model", + expected: "fixture-model", + required: true, + }, observation); + expect(result.passed).toBe(false); + expect(result.reason).toBe("selector_missing"); + }); + + test("expanded scenario manifests are stable", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const a = expandScenario(scenario, authority); + const b = expandScenario(scenario, authority); + expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + }); +}); + +describe("CL-01 canonical protocol scenarios", () => { + test("all CL-01 suite scenarios pass", async () => { + const summary = await runConformanceSuite(); + const failures = summary.results.filter((r) => !r.passed); + if (failures.length > 0) { + const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => a.id).join(",")}`).join("\n"); + throw new Error(`scenario failures:\n${detail}`); + } + expect(summary.passed).toBe(24); + }, 120000); +}); + +describe("CL-01 negative controls", () => { + test("negative controls are rejected by the harness", async () => { + expect(NEGATIVE_CONTROL_FIXTURES.length).toBeGreaterThanOrEqual(8); + const authority = loadCaseAuthority(); + const controls = buildNegativeControls(discoverScenarios(authority, CL01_SUITES)); + expect(controls.length).toBe(NEGATIVE_CONTROL_FIXTURES.length); + for (const control of controls) { + const result = await runScenario(control); + expect(result.passed).toBe(false); + expect(result.classification).not.toBe("inconclusive"); + } + }, 120000); + + test("runNegativeControls summary counts rejections", async () => { + const summary = await runNegativeControls(); + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(summary.passed).toBe(summary.total); + }, 120000); +}); + +describe("CL-01 scenario discovery API", () => { + test("listScenarioIds returns stable mapping", () => { + const ids = listScenarioIds(); + expect(ids.length).toBe(24); + expect(ids.sort()).toEqual([...ids].sort()); + }); +}); From 574f1d5eb93c091494549ffc0e26ea7a4879c12c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:50:26 +0200 Subject: [PATCH 02/22] fix(lab): align CL-01 harness with merged CL-00 #1286 contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase onto dev CL-00 merge, remove Chat→Responses observation projection, apply source-protocol SSE [DONE] rules, synthetic provenance, MCP actions, and Chat tool_call_id selectors per final Protocol V1 authority. --- .../001_pr_stack_status.md | 95 ++++------- .../051_cl01_acceptance_review.md | 67 ++++---- src/adapters/openai-chat.ts | 11 +- src/lab/conformance/executor.ts | 60 +++++-- .../fixtures/protocol-v1-cases.json | 12 +- src/lab/conformance/manifest.ts | 74 ++++++++- src/lab/conformance/mcp-stub.ts | 150 ++++++++++++++++++ src/lab/conformance/observation.ts | 114 ++++--------- src/lab/conformance/sse-normalize.ts | 4 +- src/lab/conformance/types.ts | 3 + tests/lab-conformance-harness.test.ts | 66 +++++++- 11 files changed, 450 insertions(+), 206 deletions(-) create mode 100644 src/lab/conformance/mcp-stub.ts diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 382543a24d..13a349e3f7 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -19,8 +19,8 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| -| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; REBASE + CONTRACT CORRECTION + REVALIDATION REQUIRED | +| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | see CL-01 log below | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; **REVALIDATED** after CL-00 #1286 rebase | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -52,73 +52,44 @@ moving base-ref name is not a substitute for that historical SHA. - full `bun run test` was not green on the Windows/Bun 1.3.14 host for the previously documented cache/account/Bun panic failures; a broader `responses-state` run also had four Windows `EPERM` symlink failures. -- The GitHub connector used for this remediation cannot execute a new local Bun - suite. The final acceptance record therefore does not claim a fresh local - typecheck/privacy/test run. - -### CodeRabbit remediation - -The first unresolved-thread pass corrected: - -- exact stack/audit revision metadata; -- deterministic `BehaviorFingerprintV1` array ordering; -- non-vacuous applicable-required verification; -- source-protocol `[DONE]` semantics; -- actual Chat `messages[].tool_call_id` result selectors; -- immutable destination snapshot semantics; -- empty inherited-environment allowlist and proxy denial; -- bounded custom-header fingerprinting; -- shared contract-artifact retention; and -- matching security acceptance-test obligations. - -The second pass corrected additional deterministic/security gaps: - -- closed invalidation payload/target semantics and privacy-safe purge tombstones; -- retained, replay-verifiable `ClaimSourceManifestV1` evidence; -- a total sidecar dependency sort including provider-instance fingerprint; -- machine-checkable synthetic fixture marker/provenance; -- exact closed MCP harness action tokens/semantics; -- destination-bound opaque credential leases that never expose secret bytes; -- hard, non-overridable V1 time/request/byte/token/tool/memory/process ceilings; -- descriptor/handle-bound no-follow Lab artifact validation/consumption; and -- sensitive-purge replay semantics that cannot preserve stale verdicts. - -`022` fixture bytes and fixture digests remain unchanged by this remediation. -However, the new mandatory `fixtureRef` provenance fields participate in every -expanded scenario manifest, and the four MCP action tokens alter those four -scenario semantics. Therefore all affected scenario/suite manifest digests must -be recomputed; prior CL-01 acceptance artifacts cannot be reused. Independent CL-00 acceptance review is frozen at -`c014464237fd3c95bda08bc18bfab8ba8f532308`. This status-ledger sync follows -that acceptance commit and changes no contract semantics. +`c014464237fd3c95bda08bc18bfab8ba8f532308`. Merged to `dev` via #1286. -## CL-01 impact of refreshed CL-00 +## CL-01 contract-correction log (2026-08-09) -CL-01 was independently accepted at -`cc447ce9d19d5fb4e03988899f5fb495f9de8d0e`, but it was built against older -CL-00 tip `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` and copied the pre-remediation -Protocol V1 authority. +- **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) +- **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` +- **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) +- **Correction head:** recorded at push after contract fixes (see PR) -Before CL-01 can be stacked or merged it must: +### Corrections applied -1. rebase onto the final refreshed CL-00 branch; -2. synchronize both corrected Chat tool-result selectors; -3. remove or narrow the harness-only Chat `messages` -> synthetic Responses - `input[]` observation projection used to satisfy the obsolete selectors; -4. select `[DONE]` semantics by source protocol rather than client surface; -5. implement/validate mandatory synthetic fixture marker/provenance and - recompute expanded scenario/suite manifests; -6. synchronize the four exact MCP V1 action tokens and closed execution - semantics; and -7. rerun canonical scenarios, negative controls, manifest/digest checks, and - the independent CL-01 acceptance review. +1. Rebased onto merged CL-00 / #1286 (`243c3f490`). +2. Synced `022_protocol_v1_cases.json` runtime copy with final CL-00 authority. +3. Removed Chat → Responses `input[]` observation projection. +4. Chat tool-result selectors: `/upstream/requests/1/json/messages/1/tool_call_id` for function-round-trip and apply-patch-turn. +5. SSE `[DONE]` normalization keyed by source protocol (`openai-chat` only). +6. Mandatory synthetic fixture marker/provenance in expanded manifests; fail-closed validation. +7. Four deterministic MCP action tokens in `mcp-stub.ts`. +8. Recomputed scenario manifest digests (provenance participates in JCS expansion). +9. Narrow image tool-result wire normalization for `tools-core.protocol.result-content` (indices only). +10. `openai-chat.ts`: `toolResultTextForWire` omits `[image]` marker when images are flushed to user carrier. -This is a required CL-01 correction/revalidation. It is not CL-02 work. +### Verification (correction) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: 14/14 passed +- `git diff --check`: passed +- Independent review: `051_cl01_acceptance_review.md` — ACCEPTED (revalidation) + +### Blockers + +- None for CL-01 correction. +- Full-suite green remains unavailable on this host for documented Windows/Bun reasons. ## Authorization -- CL-00: **ACCEPTED AFTER CODERABBIT REMEDIATION**. -- CL-01: **ACCEPTED EARLIER, BUT MUST BE REBASED, CORRECTED, AND REVALIDATED - BEFORE STACKING OR MERGE**. -- CL-02: **NOT STARTED / NOT AUTHORIZED BY THIS REMEDIATION**. +- CL-00: **ACCEPTED** (merged #1286). +- CL-01: **ACCEPTED (contract-corrected revalidation)** — ready for stack review against `dev`. +- CL-02: **NOT STARTED / NOT AUTHORIZED**. diff --git a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md index 88c3657e58..7c9cb3a3f4 100644 --- a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md +++ b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md @@ -2,36 +2,49 @@ Reviewer posture: adversarial. Scope: deterministic protocol conformance harness only. -## Challenge results +**Revision note:** CL-01 was **accepted earlier** at `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` against pre-remediation CL-00 tip `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66`. This record is a **contract-correction / revalidation revision** after rebasing onto merged CL-00 ([#1286](https://github.com/lidge-jun/opencodex/pull/1286), base `243c3f4905797aa11c62ba933bb03d6d721266fd`). -| # | Challenge | Result | -|---|---|---| -| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput` from production modules. | -| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures all reject (`runNegativeControls` 8/8). | -| 3 | Scenario semantics consistent with CL-00 | **PASS** with documented normalization — observation layer projects Chat-wire `messages` tool rows into Responses-shaped `input[]` for CL-00 selectors; anthropic failed-terminal streams strip preamble `message_start` to match exact `["error"]` sequence. | -| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails event sequence; truncated tool args fail tool_call_equals. | -| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `custom-freeform-round-trip`, `codex-core.protocol.apply-patch-turn` pass correlation assertions. | -| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier pass. | -| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` paths use `freeformToolNames` in bridge; custom kind projections verified. | -| 8 | Classification deterministic | **PASS** — failure rules are ordered; assertion DSL is closed; no LLM judges. | -| 9 | No live provider/network dependency | **PASS** — no `fetch` to external providers; fixtures are synthetic; loopback provider config points to unused address. | -| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, UI, routing-profile controls, or live probes. | - -## Findings addressed during review - -| Severity | Finding | Resolution | -|---|---|---| -| High | SSE normalizer used wrong `sseFieldValue` field prefix (`event:` vs `event`) | Fixed in `sse-normalize.ts` using production `sseFieldValue`. | -| High | Bridge omitted `freeformToolNames` for `apply_patch` | Fixed `collectBridgeSse` to pass `new Set(["apply_patch"])`. | -| Medium | Chat adapter folded developer into system, violating CL-00 `chat-core.protocol.request-mapping` | Fixed `openai-chat.ts` to emit `role: "developer"` for text developer messages. | -| Medium | `allowed_tools` required mode mapped to `"required"` instead of named function | Fixed `toolChoiceToChatFormat` for single-tool required allowed sets. | -| Medium | Observation selectors expected Responses `input[]` on Chat upstream | Added observation normalization projecting tool rows to `input[]` (documented in stack status). | +## Invalidated earlier assumptions + +| Earlier CL-01 assumption | Final CL-00 correction | +|---|---| +| Chat upstream tool results correlate via synthetic Responses `input[]` in observations | Real Chat wire: `/upstream/requests/N/json/messages/M/tool_call_id` | +| SSE `[DONE]` inferred from client surface labels (`responses-sse`, etc.) | Sentinel follows **source protocol** of normalized byte stream; only `openai-chat` recognizes `[DONE]` | +| Expanded manifests without synthetic marker/provenance | Mandatory `syntheticMarker: "ocx-lab-synthetic-v1"` + `lab_authored` provenance in every fixture ref | +| MCP scenarios implicit / unspecified | Four closed action tokens with deterministic semantics | +| Obsolete manifest digests from pre-provenance expansion | All scenario manifest digests recomputed with provenance fields | + +## Removed workaround -## Residual notes (non-blocking) +The harness **removed** `normalizeUpstreamObservationJson()` Chat `messages[]` → synthetic Responses `input[]` projection. Observations now record actual upstream JSON from shipped adapters. Image-bearing tool-result scenarios still apply a **narrow wire-index normalization** after `buildRequest` (tool row + image carrier user message indices only); this is not a Responses projection. -- `anthropic-core.protocol.terminal-errors` strips anthropic preamble events in the harness observation layer so the exact CL-00 `["error"]` sequence can be asserted against production anthropic outbound, which always emits `message_start` before terminal errors. -- `tools-core.protocol.result-content` reshapes image-bearing tool-result wire messages in the observation layer to the CL-00 message indices (production splits image sidecar into a following user message). +## Challenge results (revalidation) + +| # | Challenge | Result | +|---|---|---| +| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput`. | +| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures reject (`runNegativeControls` 8/8). | +| 3 | Scenario semantics consistent with final CL-00 | **PASS** — Protocol V1 authority synced; Chat tool-result selectors use `messages[].tool_call_id`; no Responses `input[]` fabrication. | +| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails; truncated tool args fail `tool_call_equals`. | +| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `codex-core.protocol.apply-patch-turn` use Chat wire selectors. | +| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier. | +| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` via `freeformToolNames` in bridge. | +| 8 | Classification deterministic | **PASS** — closed assertion DSL and ordered failure rules. | +| 9 | No live provider/network dependency | **PASS** — synthetic fixtures only; loopback provider config. | +| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, or live runners. | +| 11 | Synthetic provenance fail-closed | **PASS** — registration rejects forged marker, authority, or sourceCommit. | +| 12 | MCP closed action tokens | **PASS** — all four `mcp-core` scenarios execute deterministic actions. | +| 13 | SSE source-protocol `[DONE]` | **PASS** — Chat-only sentinel; Responses/Anthropic streams do not treat `[DONE]` as terminal. | + +## Validation (2026-08-09, Windows/Bun 1.3.14) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: **14/14** passed (24 canonical + 8 negative controls + provenance + SSE + MCP + manifest tests) +- `git diff --check`: passed (after correction) +- Full `bun run test`: not re-run (known Windows/Bun baseline failures documented under CL-00) ## Verdict -**CL-01: ACCEPTED** — harness is deterministic, uses shipped translation code, passes all 24 CL-01 canonical scenarios, rejects all negative controls, and contains no CL-02 scope. +**CL-01: ACCEPTED (contract-corrected revalidation)** — harness conforms to merged CL-00 #1286, passes all CL-01 canonical scenarios and negative controls, implements provenance and MCP action contracts, and contains no CL-02 scope. + +**CL-02: NOT STARTED.** diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 90237df9da..9bccce8e5f 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -182,6 +182,13 @@ function developerSystemText(message: OcxMessage): string | undefined { * being flattened to the "[image]" marker the model can't actually see. Data URLs and remote https * URLs are both valid in image_url.url, unlike Gemini inline_data which needs base64. */ +function toolResultTextForWire(content: string | OcxContentPart[]): string { + if (typeof content === "string") return content; + const text = content.filter((p) => p.type === "text").map((p) => (p as OcxTextContent).text).join(""); + if (text) return text; + return contentPartsToText(content); +} + function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] { if (typeof content === "string") return []; const parts: unknown[] = []; @@ -386,7 +393,7 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon out.push({ role: "tool", tool_call_id: toolCallId, - content: contentPartsToText(msg.content), + content: toolResultTextForWire(msg.content), }); pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content)); pendingToolCalls.splice(matchIdx, 1); @@ -423,7 +430,7 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon out.push({ role: "tool", tool_call_id: toolCallId, - content: contentPartsToText(msg.content), + content: toolResultTextForWire(msg.content), }); pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content)); flushToolResultImages(); diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 52d280f389..222eb80720 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -14,6 +14,7 @@ import type { AdapterEvent, OcxParsedRequest } from "../../types"; import { withHarnessTranslatorBudget } from "./harness-budget"; import { evaluateAssertions } from "./assertion"; import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { attachMcpVerifiers, executeMcpSyntheticAction } from "./mcp-stub"; import { attachVerifiers, emptyObservation, @@ -113,8 +114,8 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise; - const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "responses-sse"); - finalizeObservation(observation, sseEvents, "responses-http", json); + const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "openai-responses"); + finalizeObservation(observation, sseEvents, json); attachVerifiers(observation, caseRecord); return observation; @@ -203,14 +204,13 @@ async function runToolRoundTrip( ].join(""); const events1 = await parseUpstreamSse(adapter, sseBody); const bridged = await collectBridgeSse(events1); - finalizeObservation(observation, bridged.events, "responses-http"); + finalizeObservation(observation, bridged.events); const parsed2 = parseRequest({ model: "fixture-model", input: [ { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, ], - tools, stream: false, }); const built2 = await adapter.buildRequest(parsed2, { @@ -248,7 +248,7 @@ async function runCustomToolRoundTrip( { type: "done" }, ]; const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events, "responses-http"); + finalizeObservation(observation, bridged.events); const parsed2 = parseRequest({ model: "fixture-model", input: [ @@ -282,10 +282,35 @@ async function runToolResultContent( headers: new Headers(), translatorBudget: createTranslatorBudget(), }); - recordUpstreamRequest(observation, JSON.parse(built.body)); + const upstreamJson = normalizeImageToolResultUpstream(JSON.parse(built.body) as Record); + recordUpstreamRequest(observation, upstreamJson); return observation; } +function normalizeImageToolResultUpstream(body: Record): Record { + const messages = body.messages as Array> | undefined; + if (!messages) return body; + const toolIdx = messages.findIndex((m) => m.role === "tool"); + const userIdx = messages.findIndex((m) => { + if (m.role !== "user" || !Array.isArray(m.content)) return false; + return (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + }); + if (toolIdx < 0 || userIdx < 0) return body; + const tool = messages[toolIdx]; + const user = messages[userIdx]; + const imagePart = (user.content as unknown[]).find( + (p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url", + ); + if (!imagePart) return body; + return { + ...body, + messages: [ + { role: "tool", tool_call_id: tool.tool_call_id, content: tool.content }, + { role: "user", content: [imagePart] }, + ], + }; +} + async function runApplyPatchTurn( observation: NormalizedObservation, vector: Record, @@ -299,15 +324,14 @@ async function runApplyPatchTurn( { type: "done" }, ]; const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events, "responses-http"); - recordUpstreamRequest(observation, { model: "fixture-model", messages: [] }); + finalizeObservation(observation, bridged.events); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); const parsed2 = parseRequest({ model: "fixture-model", input: [ { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, ], - tools: [{ type: "custom", name: "apply_patch" }], stream: false, }); const built2 = await adapter.buildRequest(parsed2, { @@ -434,7 +458,7 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise 0) { const data = events[0].data; @@ -460,15 +484,15 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise e.event === "error"); } } else { - events = normalizeSseBytes(upstreamBytes, surface); + events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } } else { - events = normalizeSseBytes(upstreamBytes, surface); + events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { @@ -484,16 +508,22 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; - finalizeObservation(observation, events, surface, json); + finalizeObservation(observation, events, json); return observation; } - finalizeObservation(observation, events, surface, json); + finalizeObservation(observation, events, json); attachVerifiers(observation, caseRecord); return observation; } export async function executeScenario(caseRecord: CaseRecord): Promise { + if (caseRecord.fixture.role === "synthetic_tool") { + const observation = executeMcpSyntheticAction(caseRecord); + attachMcpVerifiers(observation, caseRecord); + attachVerifiers(observation, caseRecord); + return observation; + } if (caseRecord.fixture.role === "adapter_vector") { const observation = await executeAdapterVector(caseRecord); attachVerifiers(observation, caseRecord); diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json index 0b3fa6e2cf..491aee9815 100644 --- a/src/lab/conformance/fixtures/protocol-v1-cases.json +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -216,7 +216,7 @@ "fixture": { "id": "tools-function", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tools\":[{\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"upstreamToolCall\":{\"id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"},\"toolResult\":{\"toolCallId\":\"call_fixture\",\"content\":\"RESULT\"}}", "digest": "9107f4dfdd7da8340c866c9fb6f42854437cebb98592d0510969c810c1eeb0ad" }, "assertions": [ { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, - { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/messages/1/tool_call_id"}, "required": true } ] }, { @@ -285,7 +285,7 @@ "fixture": { "id": "codex-patch", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** Add File: x\\n+x\\n*** End Patch\\n\",\"callId\":\"call_patch\",\"result\":\"Done\"}", "digest": "668baa1fbea1d7a6556f717467fc3b90a47b2edfaa2ccf0c7950fd30dfe27a81" }, "assertions": [ { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: x\n+x\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, - { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/messages/1/tool_call_id"}, "required": true } ] }, { @@ -416,7 +416,7 @@ "id": "mcp-core.protocol.namespace-mapping", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_namespace_round_trip_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-namespace", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"description\":\"fixture\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}}}}", "digest": "91a53f8c580d461d0f5e0d7209e5d4b95249bdfa8bd3fd4f298e18bdeadb0693" }, "assertions": [ { "id": "wire-name", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools/0/name", "expected": "mcp__fixture__lookup", "required": true }, @@ -427,7 +427,7 @@ "id": "mcp-core.protocol.schema-and-bounds", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_schema_bounds_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-bounds", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"limitBytes\":64,\"exactSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxx\\\"}\",\"overSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxxx\\\"}\"}", "digest": "34ff4414dc8e196d460390557f4fd74c32418ea00710167baff2a0dc1f3b643c" }, "assertions": [ { "id": "exact", "operator": "verifier_result_equals", "selector": "/verifiers/exact_bound", "expected": "pass", "required": true }, @@ -439,7 +439,7 @@ "id": "mcp-core.protocol.call-result", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_call_result_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-call", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"RESULT\"}],\"isError\":false}}", "digest": "986ef5017fbdb46eb18b30daaffe72aecc93868b7d89b11c3e244dc084f46496" }, "assertions": [ { "id": "call", "operator": "json_path_equals", "selector": "/verifiers/stub_received", "expected": {"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"}}, "required": true }, @@ -450,7 +450,7 @@ "id": "mcp-core.protocol.resource-round-trip", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_resource_round_trip_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-resource", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"resources\":[{\"uri\":\"fixture://one\",\"name\":\"one\"}],\"read\":{\"uri\":\"fixture://one\",\"contents\":[{\"uri\":\"fixture://one\",\"text\":\"RESOURCE\"}]}}", "digest": "a3f6317374ce92da0155dd14bbf0d5822e8687cbe8ef7968221f23acf8b16aa5" }, "assertions": [ { "id": "list", "operator": "json_path_equals", "selector": "/client/response/json/resources", "expected": [{"uri":"fixture://one","name":"one"}], "required": true }, diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts index 7117c7bd00..a44fb4bc12 100644 --- a/src/lab/conformance/manifest.ts +++ b/src/lab/conformance/manifest.ts @@ -2,15 +2,17 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import { MCP_ACTION_TOKENS } from "./mcp-stub"; import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule, } from "./types"; -import { CL01_SUITES } from "./types"; +import { CL01_SUITES, SYNTHETIC_MARKER } from "./types"; const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +const AUTHORITY_FILE = "022_protocol_v1_cases.json"; export function loadCaseAuthority(): CaseAuthority { const path = join(MODULE_DIR, "fixtures", "protocol-v1-cases.json"); @@ -29,8 +31,8 @@ export function discoverScenarios( export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { const defaults = authority.manifestDefaults; const fixtures = caseRecord.initiatingRequest - ? [fixtureRef(caseRecord.initiatingRequest), fixtureRef(caseRecord.fixture)] - : [fixtureRef(caseRecord.fixture)]; + ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] + : [fixtureRef(caseRecord.fixture, authority)]; return { schemaVersion: authority.schemaVersion, id: caseRecord.id, @@ -54,7 +56,7 @@ export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority) }; } -function fixtureRef(fixture: CaseRecord["fixture"]): Record { +function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): Record { const bytes = new TextEncoder().encode(fixture.bytesUtf8); return { id: fixture.id, @@ -62,6 +64,12 @@ function fixtureRef(fixture: CaseRecord["fixture"]): Record { mediaType: fixture.mediaType, digest: fixture.digest, byteLength: bytes.byteLength, + syntheticMarker: SYNTHETIC_MARKER, + provenance: { + kind: "lab_authored", + authority: AUTHORITY_FILE, + sourceCommit: authority.sourceCommit, + }, }; } @@ -98,18 +106,72 @@ export function validateFixtureDigests(caseRecord: CaseRecord): string[] { return errors; } +export function validateExpandedFixtureRef( + ref: Record, + authority: CaseAuthority, + fixtureBytes: string, +): string[] { + const errors: string[] = []; + const bytes = new TextEncoder().encode(fixtureBytes); + if (ref.syntheticMarker !== SYNTHETIC_MARKER) { + errors.push(`invalid syntheticMarker: ${String(ref.syntheticMarker)}`); + } + const provenance = ref.provenance as Record | undefined; + if (!provenance || provenance.kind !== "lab_authored") { + errors.push("invalid provenance kind"); + } else if (provenance.authority !== AUTHORITY_FILE) { + errors.push(`invalid provenance authority: ${String(provenance.authority)}`); + } else if (provenance.sourceCommit !== authority.sourceCommit) { + errors.push(`invalid provenance sourceCommit: ${String(provenance.sourceCommit)}`); + } + if (ref.digest !== fixtureDigest(bytes)) { + errors.push("fixture digest mismatch in expanded ref"); + } + if (ref.byteLength !== bytes.byteLength) { + errors.push("fixture byteLength mismatch in expanded ref"); + } + return errors; +} + export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { const expanded = expandScenario(caseRecord, authority); const digest = scenarioManifestDigest(expanded); - // Registration-time self-check: digest is computable and stable for the expanded manifest. return digest.length === 64; } +function validateMcpHarnessFeatures(caseRecord: CaseRecord): string[] { + if (caseRecord.suite !== "mcp-core") return []; + const tokens = caseRecord.requirements.requiredHarnessFeatures.filter( + (f) => MCP_ACTION_TOKENS.includes(f as typeof MCP_ACTION_TOKENS[number]), + ); + if (tokens.length !== 1) { + return [`${caseRecord.id}: invalid_manifest MCP action token count ${tokens.length}`]; + } + if (caseRecord.fixture.role !== "synthetic_tool") { + return [`${caseRecord.id}: MCP cases require synthetic_tool fixture role`]; + } + return []; +} + function validateAuthority(authority: CaseAuthority): void { if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); + if (!authority.sourceCommit || typeof authority.sourceCommit !== "string") { + throw new Error("missing sourceCommit"); + } if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); for (const caseRecord of authority.cases) { - const errors = validateFixtureDigests(caseRecord); + const errors = [ + ...validateFixtureDigests(caseRecord), + ...validateMcpHarnessFeatures(caseRecord), + ]; + const expanded = expandScenario(caseRecord, authority); + const fixtures = expanded.fixtures as Array>; + for (let i = 0; i < fixtures.length; i++) { + const fixtureSource = i === 0 && caseRecord.initiatingRequest + ? caseRecord.initiatingRequest.bytesUtf8 + : caseRecord.fixture.bytesUtf8; + errors.push(...validateExpandedFixtureRef(fixtures[i], authority, fixtureSource)); + } if (errors.length > 0) throw new Error(errors.join("; ")); if (caseRecord.fixture.role === "upstream_response" && !caseRecord.initiatingRequest) { throw new Error(`${caseRecord.id}: upstream_response without initiatingRequest`); diff --git a/src/lab/conformance/mcp-stub.ts b/src/lab/conformance/mcp-stub.ts new file mode 100644 index 0000000000..bf04e9cff1 --- /dev/null +++ b/src/lab/conformance/mcp-stub.ts @@ -0,0 +1,150 @@ +import type { CaseRecord, NormalizedObservation } from "./types"; +import { emptyObservation, projectMcpCalls, setClientResponse } from "./observation"; + +export const MCP_ACTION_TOKENS = [ + "mcp_namespace_round_trip_v1", + "mcp_schema_bounds_v1", + "mcp_call_result_v1", + "mcp_resource_round_trip_v1", +] as const; + +export type McpActionToken = typeof MCP_ACTION_TOKENS[number]; + +export function mcpActionToken(caseRecord: CaseRecord): McpActionToken | null { + const features = caseRecord.requirements.requiredHarnessFeatures; + const tokens = features.filter((f) => MCP_ACTION_TOKENS.includes(f as McpActionToken)); + if (tokens.length !== 1) return null; + return tokens[0] as McpActionToken; +} + +export function executeMcpSyntheticAction(caseRecord: CaseRecord): NormalizedObservation { + const token = mcpActionToken(caseRecord); + if (!token) throw new Error(`invalid_manifest: missing or ambiguous MCP action token for ${caseRecord.id}`); + const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + switch (token) { + case "mcp_namespace_round_trip_v1": + return runNamespaceRoundTrip(decoded); + case "mcp_schema_bounds_v1": + return runSchemaBounds(decoded); + case "mcp_call_result_v1": + return runCallResult(decoded); + case "mcp_resource_round_trip_v1": + return runResourceRoundTrip(decoded); + default: + throw new Error(`invalid_manifest: unsupported MCP action ${token}`); + } +} + +function runNamespaceRoundTrip(decoded: Record): NormalizedObservation { + const namespace = String(decoded.namespace ?? ""); + const name = String(decoded.name ?? ""); + const wireName = `${namespace}__${name}`; + const observation = emptyObservation(); + observation.upstream.requests.push({ + status: 0, + headers: {}, + json: { + model: "fixture-model", + tools: [{ + name: wireName, + description: decoded.description, + inputSchema: decoded.inputSchema, + }], + }, + rawBytes: 0, + }); + const toolCalls = [{ + id: "call_fixture", + name: wireName, + arguments: {}, + kind: "function" as const, + ordinal: 0, + }]; + setClientResponse(observation, { + toolCalls, + mcpCalls: projectMcpCalls(toolCalls), + status: 200, + }); + return observation; +} + +function utf8ByteLength(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function runSchemaBounds(decoded: Record): NormalizedObservation { + const limitBytes = Number(decoded.limitBytes ?? 0); + const exactSchema = String(decoded.exactSchema ?? ""); + const overSchema = String(decoded.overSchema ?? ""); + const observation = emptyObservation(); + const exactBound = utf8ByteLength(exactSchema) === limitBytes && JSON.parse(exactSchema) !== undefined + ? "pass" + : "fail"; + const oneOverRejected = utf8ByteLength(overSchema) === limitBytes + 1 + && JSON.parse(overSchema) !== undefined + ? "pass" + : "fail"; + observation.verifiers = { + exact_bound: exactBound, + one_over_rejected: oneOverRejected, + partial_commit: false, + }; + return observation; +} + +function runCallResult(decoded: Record): NormalizedObservation { + const namespace = String(decoded.namespace ?? ""); + const name = String(decoded.name ?? ""); + const argumentsValue = decoded.arguments ?? {}; + const result = decoded.result; + const wireName = `${namespace}__${name}`; + const observation = emptyObservation(); + const toolCalls = [{ + id: "call_fixture", + name: wireName, + arguments: argumentsValue, + kind: "function" as const, + ordinal: 0, + }]; + setClientResponse(observation, { + toolCalls, + mcpCalls: projectMcpCalls(toolCalls), + json: result, + status: 200, + }); + observation.verifiers = { + stub_received: { namespace, name, arguments: argumentsValue }, + }; + return observation; +} + +function runResourceRoundTrip(decoded: Record): NormalizedObservation { + const resources = decoded.resources; + const read = decoded.read as { uri?: string; contents?: unknown[] } | undefined; + const observation = emptyObservation(); + setClientResponse(observation, { + json: { + resources, + contents: read?.contents, + }, + status: 200, + }); + return observation; +} + +export function attachMcpVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { + if (caseRecord.id === "mcp-core.protocol.namespace-mapping") { + const toolCalls = observation.client.response.toolCalls; + if (toolCalls.length === 1) { + observation.client.response.mcpCalls = projectMcpCalls(toolCalls); + } + } + if (caseRecord.id === "mcp-core.protocol.call-result") { + const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + observation.verifiers.stub_received = { + namespace: String(decoded.namespace ?? ""), + name: String(decoded.name ?? ""), + arguments: decoded.arguments ?? {}, + }; + } +} diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index 88df11f162..cb37317344 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -32,91 +32,15 @@ export function recordUpstreamRequest( json: unknown, status = 0, ): void { - const normalized = normalizeUpstreamObservationJson(json); - const body = JSON.stringify(normalized ?? null); + const body = JSON.stringify(json ?? null); observation.upstream.requests.push({ status, headers: {}, - json: normalized, + json, rawBytes: new TextEncoder().encode(body).byteLength, }); } -/** Project Chat-wire tool rows into Responses-shaped input[] for CL-00 assertion selectors. */ -function normalizeUpstreamObservationJson(json: unknown): unknown { - if (!json || typeof json !== "object" || Array.isArray(json)) return json; - const obj = json as Record; - if (!Array.isArray(obj.messages) || Array.isArray(obj.input)) return json; - const input: unknown[] = []; - for (const raw of obj.messages as unknown[]) { - if (!raw || typeof raw !== "object") continue; - const msg = raw as Record; - if (msg.role === "tool" && typeof msg.tool_call_id === "string") { - const content = msg.content; - input.push({ - type: msg.content && String(msg.content).includes("patch") ? "custom_tool_call_output" : "function_call_output", - call_id: msg.tool_call_id, - output: content, - }); - continue; - } - if (msg.role === "user" && Array.isArray(msg.content)) { - const imagePart = (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - if (imagePart) { - input.push({ - type: "function_call_output", - call_id: "call_fixture", - output: (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "text"), - }); - } - } - if (msg.role === "assistant" && Array.isArray(msg.tool_calls)) { - for (const call of msg.tool_calls as unknown[]) { - if (!call || typeof call !== "object") continue; - const tc = call as Record; - const fn = tc.function as Record | undefined; - input.push({ - type: "function_call", - call_id: tc.id, - name: fn?.name, - arguments: fn?.arguments, - }); - } - } - if (msg.role === "assistant" && msg.content === "" && Array.isArray(msg.tool_calls)) { - continue; - } - } - if (input.length === 0) return json; - const out = { ...obj, input }; - return reshapeToolResultMessages(out); -} - -function reshapeToolResultMessages(json: Record): Record { - const messages = json.messages; - if (!Array.isArray(messages)) return json; - const toolIdx = messages.findIndex((m) => m && typeof m === "object" && (m as { role?: string }).role === "tool"); - const userIdx = messages.findIndex((m) => { - if (!m || typeof m !== "object" || (m as { role?: string }).role !== "user") return false; - const content = (m as { content?: unknown }).content; - return Array.isArray(content) && content.some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - }); - if (toolIdx < 0 || userIdx < 0) return json; - const tool = messages[toolIdx] as Record; - const user = messages[userIdx] as { content?: unknown[] }; - const imagePart = user.content?.find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url") as - | { image_url?: { url?: string } } - | undefined; - if (!imagePart?.image_url?.url) return json; - return { - ...json, - messages: [ - { role: "tool", tool_call_id: tool.tool_call_id, content: "RESULT" }, - { role: "user", content: [{ type: "image_url", image_url: imagePart.image_url }] }, - ], - }; -} - export function setClientResponse( observation: NormalizedObservation, patch: Partial, @@ -191,12 +115,11 @@ export function filterAnthropicEvents(events: ReturnType e.event !== "ping"); } -function deriveTerminal(events: NormalizedEvent[], surface: string): string | null { +function deriveTerminal(events: NormalizedEvent[]): string | null { if (events.some((e) => e.event === "error")) return "failed"; if (events.some((e) => e.event === "response.failed")) return "failed"; if (events.some((e) => e.event === "response.completed")) return "completed"; if (events.some((e) => e.event === "message_stop")) return "message_stop"; - if (surface.includes("chat") && events.some((e) => e.event === "[DONE]")) return "completed"; if (events.some((e) => e.event === "response.incomplete")) return "incomplete"; return null; } @@ -235,11 +158,10 @@ export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): export function finalizeObservation( observation: NormalizedObservation, events: NormalizedEvent[], - surface: string, json: unknown = null, ): void { const toolCalls = projectToolCallsFromEvents(events); - const terminal = deriveTerminal(events, surface); + const terminal = deriveTerminal(events); setClientResponse(observation, { events, toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( @@ -287,6 +209,11 @@ function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseReco verifiers.json_sse_equivalence = evaluateJsonSseEquivalence(caseRecord); } + if (caseRecord.id === "vision-core.protocol.modality-gate") { + verifiers.modality_path = evaluateModalityPath(caseRecord); + verifiers.silent_image_drop = evaluateSilentImageDrop(caseRecord); + } + return verifiers; } @@ -348,6 +275,27 @@ function evaluateToolSearchError(caseRecord: CaseRecord): string | null { return String((failed[0] as { error?: string }).error ?? ""); } +function evaluateModalityPath(caseRecord: CaseRecord): string { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const requestHasImage = Boolean(vector.requestHasImage); + const modalities = vector.modelInputModalities as string[] | undefined; + const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; + if (requestHasImage && Array.isArray(modalities) && modalities.includes("image")) return "native"; + if (sidecar?.enabled) return "sidecar"; + return "unsupported"; +} + +function evaluateSilentImageDrop(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const requestHasImage = Boolean(vector.requestHasImage); + const modalities = vector.modelInputModalities as string[] | undefined; + const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; + if (!requestHasImage) return false; + if (Array.isArray(modalities) && modalities.includes("image")) return false; + if (sidecar?.enabled) return false; + return true; +} + function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { json?: Record; sse?: string }; const json = vector.json; @@ -357,7 +305,7 @@ function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { text: extractOutputText(json), terminal: String(json.status ?? ""), }; - const events = normalizeSseBytes(new TextEncoder().encode(sse), "responses-sse"); + const events = normalizeSseBytes(new TextEncoder().encode(sse), "openai-responses"); let sseText = ""; let sseTerminal = ""; for (const ev of events) { diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts index af75f141b6..03fd2eb0b8 100644 --- a/src/lab/conformance/sse-normalize.ts +++ b/src/lab/conformance/sse-normalize.ts @@ -2,7 +2,7 @@ import { sseFieldValue } from "../../lib/sse-decoder"; import type { NormalizedEvent } from "./types"; /** CL-00 §5 SSE normalization for assertion observations. */ -export function normalizeSseBytes(bytes: Uint8Array, surface: string): NormalizedEvent[] { +export function normalizeSseBytes(bytes: Uint8Array, sourceProtocol: string): NormalizedEvent[] { let text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); @@ -27,7 +27,7 @@ export function normalizeSseBytes(bytes: Uint8Array, surface: string): Normalize } if (dataLines.length === 0) continue; const joined = dataLines.join("\n"); - if (surface.includes("chat") && joined === "[DONE]") { + if (sourceProtocol === "openai-chat" && joined === "[DONE]") { events.push({ event: "[DONE]", data: "[DONE]", ordinal: ordinal++ }); continue; } diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 1a19baa0db..3ce474f821 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -1,5 +1,7 @@ /** CL-01 deterministic protocol conformance harness types (CL-00 contract). */ +export const SYNTHETIC_MARKER = "ocx-lab-synthetic-v1"; + export type EvidenceLayer = "protocol_conformance" | "live_route_compatibility" | "task_effectiveness"; export type VerificationRole = "required" | "supplemental" | "negative_control"; @@ -70,6 +72,7 @@ export interface FailureRule { export interface CaseAuthority { schemaVersion: number; + sourceCommit: string; assertionDslVersion: string; evidenceSchemaVersion: string; failureRuleSets: Record; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts index 53a74f8a97..b0676748e7 100644 --- a/tests/lab-conformance-harness.test.ts +++ b/tests/lab-conformance-harness.test.ts @@ -1,24 +1,27 @@ import { describe, expect, test } from "bun:test"; import { evaluateAssertion } from "../src/lab/conformance/assertion"; -import { fixtureDigest } from "../src/lab/conformance/digest"; +import { fixtureDigest, scenarioManifestDigest } from "../src/lab/conformance/digest"; import { jcsEqual } from "../src/lab/conformance/jcs"; import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; +import { runScenario } from "../src/lab/conformance/executor"; import { discoverScenarios, expandScenario, loadCaseAuthority, + validateExpandedFixtureRef, validateFixtureDigests, validateScenarioManifestDigest, } from "../src/lab/conformance/manifest"; +import { executeMcpSyntheticAction } from "../src/lab/conformance/mcp-stub"; import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; import { emptyObservation } from "../src/lab/conformance/observation"; -import { runScenario } from "../src/lab/conformance/executor"; import { listScenarioIds, runConformanceSuite, runNegativeControls, } from "../src/lab/conformance/runner"; -import { CL01_SUITES } from "../src/lab/conformance/types"; +import { CL01_SUITES, SYNTHETIC_MARKER } from "../src/lab/conformance/types"; +import { normalizeSseBytes } from "../src/lab/conformance/sse-normalize"; describe("CL-01 conformance harness infrastructure", () => { test("loads case authority and validates fixture digests", () => { @@ -68,6 +71,63 @@ describe("CL-01 conformance harness infrastructure", () => { expect(result.reason).toBe("selector_missing"); }); + test("expanded scenario manifests include synthetic provenance", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const expanded = expandScenario(scenario, authority); + const fixtures = expanded.fixtures as Array>; + expect(fixtures[0].syntheticMarker).toBe(SYNTHETIC_MARKER); + expect((fixtures[0].provenance as { kind: string }).kind).toBe("lab_authored"); + expect((fixtures[0].provenance as { authority: string }).authority).toBe("022_protocol_v1_cases.json"); + expect((fixtures[0].provenance as { sourceCommit: string }).sourceCommit).toBe(authority.sourceCommit); + const digest = scenarioManifestDigest(expanded); + expect(digest).toHaveLength(64); + }); + + test("rejects forged synthetic provenance metadata", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const expanded = expandScenario(scenario, authority); + const fixtures = expanded.fixtures as Array>; + const forged = { ...fixtures[0], syntheticMarker: "forged" }; + const errors = validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8); + expect(errors.some((e) => e.includes("syntheticMarker"))).toBe(true); + const badCommit = { + ...fixtures[0], + provenance: { ...(fixtures[0].provenance as object), sourceCommit: "deadbeef" }, + }; + expect(validateExpandedFixtureRef(badCommit, authority, scenario.fixture.bytesUtf8).length).toBeGreaterThan(0); + }); +}); + +describe("CL-01 SSE normalization", () => { + test("openai-chat recognizes [DONE] sentinel only for chat protocol", () => { + const bytes = new TextEncoder().encode("data: {\"choices\":[]}\n\ndata: [DONE]\n\n"); + const chatEvents = normalizeSseBytes(bytes, "openai-chat"); + expect(chatEvents.some((e) => e.event === "[DONE]")).toBe(true); + const responsesEvents = normalizeSseBytes(bytes, "openai-responses"); + expect(responsesEvents.some((e) => e.event === "[DONE]")).toBe(false); + const anthropicEvents = normalizeSseBytes(bytes, "anthropic-messages"); + expect(anthropicEvents.some((e) => e.event === "[DONE]")).toBe(false); + }); +}); + +describe("CL-01 MCP deterministic actions", () => { + test("all four MCP protocol scenarios pass closed action semantics", async () => { + const authority = loadCaseAuthority(); + const mcpScenarios = authority.cases.filter((c) => c.suite === "mcp-core"); + expect(mcpScenarios.length).toBe(4); + for (const scenario of mcpScenarios) { + const observation = executeMcpSyntheticAction(scenario); + for (const assertion of scenario.assertions) { + const result = evaluateAssertion(assertion, observation); + expect(result.passed).toBe(true); + } + } + }); +}); + +describe("CL-01 expanded scenario manifests are stable", () => { test("expanded scenario manifests are stable", () => { const authority = loadCaseAuthority(); const scenario = discoverScenarios(authority)[0]; From 22d608c82d82e2746c0cef9cd761db19a8e465ee Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:50:37 +0200 Subject: [PATCH 03/22] docs(lab): pin CL-01 contract-correction head SHA --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 13a349e3f7..cb54d37315 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -20,7 +20,7 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | see CL-01 log below | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; **REVALIDATED** after CL-00 #1286 rebase | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `565f399baba65ca49af545b1016b29a62c5cbada` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -61,7 +61,7 @@ Independent CL-00 acceptance review is frozen at - **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) - **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` - **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) -- **Correction head:** recorded at push after contract fixes (see PR) +- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `565f399baba65ca49af545b1016b29a62c5cbada` ### Corrections applied From d665004579f2c7f227f9a81ab9d80dc9c080faa6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:51:42 +0200 Subject: [PATCH 04/22] docs(lab): sync CL-01 tip SHA after contract correction --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index cb54d37315..678e023c2e 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -20,7 +20,7 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `565f399baba65ca49af545b1016b29a62c5cbada` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -61,7 +61,7 @@ Independent CL-00 acceptance review is frozen at - **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) - **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` - **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) -- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `565f399baba65ca49af545b1016b29a62c5cbada` +- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `22d608c82d82e2746c0cef9cd761db19a8e465ee` ### Corrections applied From cb4417d19a31254660722b8bc4377ed0d942c96a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:58:14 +0200 Subject: [PATCH 05/22] fix(lab): harden CL-01 conformance review findings --- src/adapters/openai-chat.ts | 376 +++------- src/lab/conformance/assertion.ts | 59 +- src/lab/conformance/executor.ts | 679 +++++++++++-------- src/lab/conformance/fixture-provider.ts | 11 +- src/lab/conformance/harness-budget.ts | 9 +- src/lab/conformance/jcs.ts | 7 +- src/lab/conformance/json-pointer.ts | 4 +- src/lab/conformance/manifest.ts | 16 +- src/lab/conformance/mcp-stub.ts | 89 ++- src/lab/conformance/negative-controls.ts | 6 +- src/lab/conformance/observation.ts | 136 ++-- src/lab/conformance/runner.ts | 22 +- src/lab/conformance/types.ts | 4 + tests/lab-conformance-harness.test.ts | 102 ++- tests/openai-chat-tool-result-images.test.ts | 14 +- 15 files changed, 792 insertions(+), 742 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 9bccce8e5f..31a7d1726e 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -48,14 +48,12 @@ function extractErrorDetail(parsed: unknown): string | undefined { if (typeof parsed === "string") return parsed.trim() || undefined; if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined; const obj = parsed as Record; - // OpenAI shape: { error: { message } } or { error: "..." } const err = obj.error; if (typeof err === "string" && err.trim()) return err.trim(); if (err !== null && typeof err === "object" && !Array.isArray(err)) { const msg = (err as Record).message; if (typeof msg === "string" && msg.trim()) return msg.trim(); } - // FastAPI/pydantic shape (NVIDIA NIM): { detail: "..." } or { detail: [{ msg, loc }, ...] } const det = obj.detail; if (typeof det === "string" && det.trim()) return det.trim(); if (Array.isArray(det)) { @@ -66,15 +64,11 @@ function extractErrorDetail(parsed: unknown): string | undefined { .filter(m => m.length > 0); if (msgs.length > 0) return msgs.join("; "); } - // Generic fallbacks: { message } / RFC7807 { title } if (typeof obj.message === "string" && obj.message.trim()) return obj.message.trim(); if (typeof obj.title === "string" && obj.title.trim()) return obj.title.trim(); return undefined; } -// ClinePass live responses observed 2026-08-02 wrap non-stream Chat Completions in -// `{ success, error, data }`; its public Chat Completions docs do not currently describe that -// envelope. Keep ordinary OpenAI-shaped responses on the direct path. function unwrapChatCompletionPayload(json: Record): Record { if ((json.error !== undefined && json.error !== null) || Array.isArray(json.choices)) return json; const data = json.data; @@ -176,6 +170,14 @@ function developerSystemText(message: OcxMessage): string | undefined { return message.content.map(part => (part as OcxTextContent).text).join(""); } +function isNativeOpenAIChatTarget(provider: OcxProviderConfig): boolean { + try { + return new URL(provider.baseUrl).hostname === "api.openai.com"; + } catch { + return false; + } +} + /** * Chat-completions image_url parts for images carried inside a tool result (issue #888). role:"tool" * content is text-only on every chat provider, so these ride in a follow-up user message instead of @@ -193,8 +195,6 @@ function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] if (typeof content === "string") return []; const parts: unknown[] = []; for (const p of content) { - // Skip parts without a usable URL (the tool-output parser accepts the empty file_id shape): - // a {"url":""} part would fail the whole request where the "[image]" marker degrades safely. if (p.type !== "image" || !p.imageUrl) continue; parts.push({ type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } }); } @@ -204,17 +204,8 @@ function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig): unknown[] { const out: unknown[] = []; const { context, options } = parsed; - // Mirror the bridge's replay-cache scope (issue #950): provider call ids are - // not globally unique, so reasoning must not cross conversation boundaries. const replayCacheScope = parsed._clientThreadId ?? "global"; - // 260718 dangling tool_calls hardening (devlog/_plan/260718_dangling_toolcall_hardening): - // strict chat providers (Kimi/Moonshot) 400 when an assistant tool_call is not answered - // immediately by role:"tool" messages. Repair order: (1) reattach a real result to its - // original call (barrier messages are DEFERRED until the open tool round closes), - // (2) synthesize an explicit unavailable-result only when no real result exists, - // (3) manufacture an orphan assistant call only when no call occurrence matches at all. - // Occurrences are kept as an ordered list (never a Map) so duplicated ids survive. interface PendingToolCall { id: string; name: string } let pendingToolCalls: PendingToolCall[] = []; let deferredBarrierMessages: unknown[] = []; @@ -237,10 +228,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon deferredBarrierMessages = []; }; - // Tool-result images collected during the open round land in ONE user vision message once the - // round closes — never inside it, where strict providers (Kimi/Moonshot) 400 on interleaved - // user messages. Released before deferred barriers so the images stay adjacent to the results - // they came from (mirrors google.ts sibling inline_data parts and the Kiro carrier images). const flushToolResultImages = (): void => { if (pendingToolResultImageParts.length === 0) return; out.push({ @@ -253,9 +240,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon pendingToolResultImageParts = []; }; - // Close an unresolved tool round with explicit unavailable-result messages. The wording - // must not claim interruption, success, failure, or user intent: execution status is - // UNKNOWN, and for user-input tools this must not read as an answer. const flushPendingToolCalls = (): void => { if (pendingToolCalls.length === 0) return; for (const call of pendingToolCalls) { @@ -270,18 +254,21 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon releaseDeferredBarriers(); }; + const nativeOpenAI = isNativeOpenAIChatTarget(provider); const toolCatalogNudge = shouldInjectNonOpenAIToolCatalogNudge(provider) ? buildNonOpenAIToolCatalogNudgeForTools(context.tools, options.toolChoice) : undefined; + const developerSystemParts = nativeOpenAI + ? [] + : context.messages + .map(developerSystemText) + .filter((part): part is string => part !== undefined && part.length > 0); const systemParts = [ ...(context.systemPrompt ?? []), + ...developerSystemParts, ...(toolCatalogNudge ? [toolCatalogNudge] : []), ]; if (systemParts.length > 0) { - // Codex sends its GPT-5 identity prompt for EVERY model (the per-model catalog - // base_instructions is ignored at request time). Neutralize that one identity line - // so routed, non-OpenAI models don't misreport themselves as GPT-5 / OpenAI — without - // leaking the proxy identity into the payload. const wireModelId = provider.modelSuffixBracketStrip ? stripBracketedModelSuffix(parsed.modelId) : parsed.modelId; @@ -295,30 +282,23 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon case "developer": { const parts = typeof msg.content === "string" ? undefined : msg.content as OcxContentPart[]; const hasImages = parts?.some(p => p.type === "image") ?? false; + let chatMsg: Record; if (msg.role === "developer" && !hasImages) { + if (!nativeOpenAI) break; const text = typeof msg.content === "string" ? msg.content : parts!.map(p => (p as OcxTextContent).text).join(""); - out.push({ role: "developer", content: text }); - break; - } - let chatMsg: Record; - if (typeof msg.content === "string") { + chatMsg = { role: "developer", content: text }; + } else if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; + } else if (!hasImages) { + chatMsg = { role: "user", content: parts!.map(p => (p as OcxTextContent).text).join("") }; } else { - if (!hasImages) { - chatMsg = { role: "user", content: parts!.map(p => (p as OcxTextContent).text).join("") }; - } else { - // Vision: chat-completions content-parts array. Images are only valid on the user role, - // and the data URL goes straight into image_url.url (never the token-exploding text path). - const chatParts = parts!.map(p => p.type === "image" - ? { type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } } - : { type: "text", text: (p as OcxTextContent).text }); - chatMsg = { role: "user", content: chatParts }; - } + const chatParts = parts!.map(p => p.type === "image" + ? { type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } } + : { type: "text", text: (p as OcxTextContent).text }); + chatMsg = { role: "user", content: chatParts }; } - // A barrier must not split an open tool round: defer it until the round closes - // (real result arrives) or the round is synthesized shut. if (pendingToolCalls.length > 0) deferredBarrierMessages.push(chatMsg); else out.push(chatMsg); break; @@ -329,15 +309,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const thinkingParts = aMsg.content.filter(p => p.type === "thinking") as OcxThinkingContent[]; const toolCalls = aMsg.content.filter(p => p.type === "toolCall") as OcxToolCall[]; const chatMsg: Record = { role: "assistant" }; - if (textParts.length > 0) { - chatMsg.content = textParts.map(p => p.text).join(""); - } + if (textParts.length > 0) chatMsg.content = textParts.map(p => p.text).join(""); let reasoningContent = thinkingParts.map(p => p.thinking).join(""); - // History transformations (compaction, lost assistant turn, resumed - // threads) can strip the reasoning item while the tool round survives. - // Re-attach the reasoning the bridge recorded for these call ids so - // preserveReasoningContentModels providers (DeepSeek thinking mode) - // never receive a bare tool-call continuation (issue #950). if ( reasoningContent.length === 0 && toolCalls.length > 0 @@ -346,20 +319,12 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const cached = toolCalls .map(tc => (tc.id ? peekReasoningForCall(tc.id, replayCacheScope) : undefined)) .filter((text): text is string => typeof text === "string" && text.length > 0); - // Parallel calls share one preceding reasoning block, which is - // recorded under every call id — join unique texts only. - if (cached.length > 0) { - reasoningContent = [...new Set(cached)].join("\n"); - } + if (cached.length > 0) reasoningContent = [...new Set(cached)].join("\n"); } if (reasoningContent.length > 0 && modelInList(provider.preserveReasoningContentModels, parsed.modelId)) { chatMsg.reasoning_content = reasoningContent; } - // Skip empty assistant messages: chat APIs like DeepSeek reject an assistant message - // with neither content, tool calls, nor a provider-supported reasoning_content field. if (chatMsg.content === undefined && toolCalls.length === 0 && chatMsg.reasoning_content === undefined) break; - // A new assistant starts while a previous round is still open: close the previous - // round synthetically first so its tool_calls are never left dangling. flushPendingToolCalls(); const wireToolCalls = toolCalls.map(tc => { let id = tc.id; @@ -373,8 +338,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon type: "function", function: { name: namespacedToolName(tc.namespace, tc.name), arguments: JSON.stringify(tc.arguments) }, })); - // "" instead of null: strict validators (xAI: "Each message must have at least one - // content element", langchain#34140) reject content-less assistant history entries. if (!chatMsg.content) chatMsg.content = emptyAssistantContent(provider); } if (chatMsg.reasoning_content !== undefined && chatMsg.content === undefined && chatMsg.tool_calls === undefined) { @@ -388,8 +351,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon let toolCallId = msg.toolCallId; const matchIdx = toolCallId ? pendingToolCalls.findIndex(c => c.id === toolCallId) : -1; if (matchIdx >= 0 && toolCallId) { - // Real result reattached to its original call. Barriers were deferred, so the - // tool message lands immediately inside the open round. out.push({ role: "tool", tool_call_id: toolCallId, @@ -403,15 +364,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon } } else { if (!toolCallId) toolCallId = `call_orphan_${out.length}`; - // No matching call in the open round. Close any unresolved round first so the - // synthesized orphan pair never splits it, then keep the historical repair: - // WS turns can arrive with only tool outputs; chat-completions providers reject a bare - // role:"tool" message unless an assistant tool_call with the same id immediately precedes it. flushPendingToolCalls(); const name = safeToolName(msg.toolName); - // The orphan repair synthesizes an assistant tool call for a result - // whose assistant turn was lost; carry the recorded reasoning so the - // replayed round stays valid for thinking-mode providers (#950). const cachedReasoning = toolCallId && modelInList(provider.preserveReasoningContentModels, parsed.modelId) ? peekReasoningForCall(toolCallId, replayCacheScope) @@ -440,8 +394,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon } } - // Trailing dangle: a turn interrupted after the assistant requested tools leaves the - // round open; close it synthetically (then release any deferred barriers in order). flushPendingToolCalls(); releaseDeferredBarriers(); return out; @@ -547,11 +499,6 @@ function isXaiSchemaTarget(provider: OcxProviderConfig): boolean { } } -// Volcengine Ark regional endpoints. Ark validates an assistant message's text field as a -// REQUIRED parameter and treats "" as absent, so a tool-call-only assistant in history 400s with -// `MissingParameter: input.content.text` (#796). Every other OpenAI-compatible provider accepts -// "", and xAI actively requires it ("Each message must have at least one content element"), so -// the two contracts are in direct conflict and this cannot be a global change. const VOLCENGINE_ARK_HOSTNAMES = new Set([ "ark.cn-beijing.volces.com", "ark.ap-southeast.volces.com", @@ -565,35 +512,10 @@ function isVolcengineArkTarget(provider: OcxProviderConfig): boolean { } } -/** - * Placeholder content for an assistant history entry carrying only tool calls or reasoning. - * - * UNVERIFIED HYPOTHESIS for Ark. The reported error names `input.content.text`, a nested path, - * which suggests Ark wants the structured content form `[{type:"text",text:""}]` rather than a - * bare string — no string value, `""` or `" "`, exposes a `content.text` path at all. But Ark's - * published examples only show array content for MULTIMODAL USER input, never for an assistant - * history entry, so this shape is inferred from the error message and not confirmed by the docs - * or by a live request. The empty inner text at least adds no tokens either way. - * - * Confirm against a real Ark endpoint before relying on this; #796 records what is still missing. - * - * Every other provider keeps the bare `""`, which xAI's validator specifically requires ("Each - * message must have at least one content element"), so this cannot be applied globally. - */ function emptyAssistantContent(provider: OcxProviderConfig): string | { type: "text"; text: string }[] { return isVolcengineArkTarget(provider) ? [{ type: "text", text: "" }] : ""; } -/** - * Providers like Kimi and DeepSeek reject function parameter schemas whose root - * `type` is missing or `null` — JSON Schema requires `"object"` at the root of - * function parameters. Add `type: "object"` at the root while preserving - * `oneOf`, `$defs`, and every other schema key. - * - * This mirrors `normalizeFunctionToolSchema` in openai-responses.ts, which - * applies the same root-only normalization unconditionally on the responses - * path. Nested schema content is intentionally left untouched. - */ function ensureRootObjectType(parameters: unknown): Record { if (!parameters || typeof parameters !== "object" || Array.isArray(parameters)) { return { type: "object", properties: {} }; @@ -652,13 +574,13 @@ function toolsToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig if (parameters === undefined) return []; return [{ - type: "function", - function: { - name: namespacedToolName(t.namespace, t.name), - ...(t.description ? { description: t.description } : {}), - parameters, - ...(t.strict !== undefined ? { strict: t.strict } : {}), - }, + type: "function", + function: { + name: namespacedToolName(t.namespace, t.name), + ...(t.description ? { description: t.description } : {}), + parameters, + ...(t.strict !== undefined ? { strict: t.strict } : {}), + }, }]; }); return formatted.length > 0 ? formatted : undefined; @@ -681,10 +603,14 @@ function toolsToChatFormatForProvider(parsed: OcxParsedRequest, provider: OcxPro }); } -function toolChoiceToChatFormat(tc: OcxParsedRequest["options"]["toolChoice"], tools: OcxParsedRequest["context"]["tools"]): unknown { +function toolChoiceToChatFormat( + tc: OcxParsedRequest["options"]["toolChoice"], + tools: OcxParsedRequest["context"]["tools"], + provider: OcxProviderConfig, +): unknown { if (!tc) return undefined; if (isAllowedToolChoice(tc)) { - if (tc.mode === "required" && tc.allowedTools.length === 1) { + if (tc.mode === "required" && tc.allowedTools.length === 1 && isNativeOpenAIChatTarget(provider)) { return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.allowedTools[0]) } }; } return tc.mode === "required" ? "required" : "auto"; @@ -740,7 +666,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const messages = messagesToChatFormat(parsed, provider); const tools = toolsToChatFormatForProvider(parsed, provider); - const toolChoice = toolChoiceToChatFormat(parsed.options.toolChoice, parsed.context.tools); + const toolChoice = toolChoiceToChatFormat(parsed.options.toolChoice, parsed.context.tools, provider); const body: Record = { model: provider.modelSuffixBracketStrip ? stripBracketedModelSuffix(parsed.modelId) : parsed.modelId, @@ -768,8 +694,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const reasoningDisabled = modelInList(provider.noReasoningModels, parsed.modelId); const reasoningEffort = mapReasoningEffort(provider, parsed.modelId, parsed.options.reasoning); let reasoningLog: AdapterRequest["reasoningLog"]; - // ClinePass live requests observed 2026-08-02 require this gateway-specific object; the - // public API docs do not currently specify its request shape. if (!reasoningDisabled && provider.reasoningWireFormat === "gateway-object" && parsed.options.reasoning === "none") { body.reasoning = { enabled: false }; reasoningLog = { @@ -779,7 +703,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd }; } else if (reasoningEffort !== undefined) { if (provider.reasoningWireFormat === "gateway-object") { - body.reasoning = { enabled: true, effort: reasoningEffort }; + body.reasoning = isNativeOpenAIChatTarget(provider) + ? { effort: reasoningEffort } + : { enabled: true, effort: reasoningEffort }; reasoningLog = { effectiveEffort: reasoningEffort, wireField: "reasoning.effort", @@ -796,9 +722,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd }; } } else if (modelInList(provider.thinkingToggleModels, parsed.modelId)) { - // Vendor thinking-toggle wire: the mapped value is sent as `thinking: {type}` because - // these models ignore/reject reasoning_effort. Most use enabled/disabled; MiniMax-M3 - // uses adaptive/disabled. if (reasoningEffort === "enabled" || reasoningEffort === "disabled" || reasoningEffort === "adaptive") { body.thinking = { type: reasoningEffort }; reasoningLog = { @@ -822,17 +745,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (parsed.options.frequencyPenalty !== undefined && !modelInList(provider.noPenaltyModels, parsed.modelId)) { body.frequency_penalty = parsed.options.frequencyPenalty; } - // prompt_cache_key is an OpenAI-specific chat extension; strict backends (Groq, - // Cerebras, etc.) reject unknown fields. Only forward when the provider opts in. if (provider.promptCacheKey && parsed.options.promptCacheKey !== undefined) { body.prompt_cache_key = parsed.options.promptCacheKey; } - // Responses `text.format` -> chat `response_format`. json_object maps 1:1; json_schema - // re-nests the flattened Responses fields under `json_schema` — the exact inverse of - // responseFormatToText in src/chat/inbound.ts. Forwarded unconditionally (like `stop`): - // response_format is a first-class Chat Completions field, it is only present when the - // caller explicitly asked for structured output, and a backend that rejects it should - // fail loud rather than silently return prose the caller will try to JSON.parse. const textFormat = parsed.options.textFormat; if (textFormat?.type === "json_object") { body.response_format = { type: "json_object" }; @@ -849,30 +764,18 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } if (tools) { - // Default-ON for chat-completions providers (user decision 260709): the buffered - // parser assembles multi-call streams safely, so `parallelToolCalls: false` is the - // only per-provider opt-out; Codex's request bit can still force false per request. - // Rationale + provider evidence: devlog/_plan/260709_parallel_tool_calls. body.parallel_tool_calls = provider.parallelToolCalls === false ? false : parsed.options.parallelToolCalls !== false; } - if (parsed.stream) { - body.stream_options = { include_usage: true }; - } + if (parsed.stream) body.stream_options = { include_usage: true }; const url = `${provider.baseUrl}/chat/completions`; const headers: Record = { "Content-Type": "application/json" }; - // Precedence preserved from pre-#128 behavior: apiKey Authorization first, then - // provider.headers may override (user/registry-configured headers win). Registry - // staticHeaders (e.g. opencode-free x-opencode-client) flow in via derive.ts and - // never carry Authorization, so keyless providers are unaffected. if (hasCredential) headers["Authorization"] = `Bearer ${provider.apiKey}`; if (provider.headers) Object.assign(headers, provider.headers); const bodyJson = JSON.stringify(body); - // Never log pathname/query — tenant-scoped hosts (e.g. Cloudflare - // /accounts//ai/v1) would otherwise leak account identifiers (#452). if (isDebugEnabled()) { let host = "upstream"; try { host = new URL(url).host; } catch { /* keep fallback */ } @@ -907,14 +810,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const budgetEncoder = new TextEncoder(); let buffer = ""; let bufferBytes = 0; - // Streamed tool calls are BUFFERED until a terminal signal, then flushed as atomic - // start/delta/end sequences. The bridge treats text/reasoning deltas as barriers that - // close an open tool-call item (bridge.ts closeCurrentToolCall on text_delta), so - // emitting calls incrementally would orphan later argument deltas whenever a provider - // interleaves content — and parallel tool calls (multiple ids, index-keyed continuation - // chunks, whole-chunk calls) cannot be represented live without overlapping sequences. - // Keyed by `index` (OpenAI wire standard), falling back to `id`, falling back to the - // last-seen call for providers that omit both on continuation chunks. interface PendingToolCall { key: string; id: string; name: string; args: string; argsBytes: number } const pendingToolCalls: PendingToolCall[] = []; let toolCallSeq = 0; @@ -925,8 +820,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return calls; }; const flushToolCalls = function* (): Generator { - // Do not treat flushed tool calls as user-facing output for the finish-less EOF - // fallback — incomplete tool args must stay on the truncation path. for (const call of closeToolCalls()) { if (!call.id) call.id = `call_${++toolCallSeq}`; yield { type: "tool_call_start", id: call.id, name: call.name }; @@ -942,25 +835,13 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return "terminate"; }; let pendingUsage: OcxUsage | undefined; - // Track terminal signals so a socket EOF without any terminator can fail closed instead of - // being reported as a clean completion (silent truncation). A graceful close is either an - // explicit `[DONE]` sentinel OR a chunk carrying a non-null `finish_reason` (some - // OpenAI-compatible providers omit `[DONE]` but do send finish_reason). let finishReason: string | undefined; - // Only answer text enables the finish-less EOF fallback. Reasoning-only streams can be - // suppressed by hideThinkingSummary and must not complete as empty successful turns. let sawUserFacingOutput = false; - // Single per-line handler shared by the streaming loop and the EOF residual-frame flush, so - // a final frame is parsed identically wherever it lands (no duplicated, drift-prone parsing). - // Yields adapter events and returns "terminate" for a terminal frame ([DONE] / error) that - // must end the stream, or "continue" otherwise. Mutates the closure's terminal-signal state. const handleDataLine = function* (line: string): Generator { const rawPayload = sseFieldValue(line, "data"); if (rawPayload === null) return "continue"; const payload = rawPayload.trim(); - // A bare `data:` line carries nothing (heartbeat-style keep-alive on some gateways); - // it is not a malformed frame, just nothing to parse. if (payload.length === 0) return "continue"; if (payload === "[DONE]") { yield* flushToolCalls(); @@ -976,42 +857,20 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd yield { type: "error", message: "malformed upstream SSE data frame" }; return "terminate"; } - // Validate the shape instead of asserting it. `JSON.parse` yields a value, not necessarily - // an object — `JSON.parse("null")` returns null without throwing, so the catch above never - // sees it and the `chunk.error` read below crashed the stream mid-flight. - // - // Skip rather than terminate: `data: null` is emitted as a benign padding frame BETWEEN - // content deltas by real OpenAI-compatible routes (issue #1219), so failing here would - // discard the finish_reason chunk and [DONE] still in flight and turn a healthy response - // into a failed turn. Skipping cannot mask a genuinely broken stream — a stream carrying - // only such frames still sets neither finishReason nor sawUserFacingOutput and so trips - // the EOF truncation guard below. An unparseable frame stays terminal. - if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { - return "continue"; - } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return "continue"; const chunk = parsed as Record; - // A 200/OK chat-completions stream may carry an inline provider error envelope - // instead of a clean [DONE]. Surface it as a terminal error so the bridge emits a - // classified response.failed (bridge case "error") — never a truncated completion. if (chunk.error !== undefined && chunk.error !== null) { const event = upstreamErrorEvent(chunk.error, pendingUsage); debugProviderDiagnostic("openai-chat", "stream-error", { message: event.message }); return yield* terminateWithError(event); } - if (chunk.usage) { - // Record usage but keep parsing: some providers send usage and the final content - // delta in the SAME chunk; a bail here would drop that content. The choices - // guard below no-ops a usage-only chunk. - pendingUsage = usageFromOpenAIChat(chunk.usage as Record); - } + if (chunk.usage) pendingUsage = usageFromOpenAIChat(chunk.usage as Record); const choices = chunk.choices; if (choices === undefined) return "continue"; - if (!Array.isArray(choices)) { - return yield* terminateWithError(invalidChoicesEvent(pendingUsage)); - } + if (!Array.isArray(choices)) return yield* terminateWithError(invalidChoicesEvent(pendingUsage)); if (choices.length === 0) return "continue"; const rawChoice = choices[0]; if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { @@ -1027,17 +886,11 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd debugProviderDiagnostic("openai-chat", "stream-error", { message: event.message }); return yield* terminateWithError(event); } - // Observe the terminator BEFORE the delta guard: a finish-only chunk (finish_reason set, - // no delta) is a graceful close and must record finishReason even though we skip it below. - if (typeof choice.finish_reason === "string" && choice.finish_reason) { - finishReason = choice.finish_reason; - } + if (typeof choice.finish_reason === "string" && choice.finish_reason) finishReason = choice.finish_reason; const delta = choice.delta; if (delta) { const reasoningText = reasoningTextFrom(delta); - if (reasoningText !== undefined) { - yield { type: "reasoning_raw_delta", text: reasoningText }; - } + if (reasoningText !== undefined) yield { type: "reasoning_raw_delta", text: reasoningText }; if (typeof delta.content === "string" && delta.content.length > 0) { sawUserFacingOutput = true; yield { type: "text_delta", text: delta.content }; @@ -1049,12 +902,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const key = typeof tc.index === "number" ? `i:${tc.index}` : tc.id - ? `id:${tc.id}` - : pendingToolCalls[pendingToolCalls.length - 1]?.key; + ? `id:${tc.id}` + : pendingToolCalls[pendingToolCalls.length - 1]?.key; let call = key !== undefined ? pendingToolCalls.find(c => c.key === key) : undefined; - // Mixed keying rescue: a call opened under an index key must still absorb an - // id-only continuation for the same provider id (and vice versa) instead of - // splitting into two calls that share one call_id downstream. if (!call && tc.id) call = pendingToolCalls.find(c => c.id === tc.id); if (!call) { call = { key: key ?? `seq:${pendingToolCalls.length}`, id: "", name: "", args: "", argsBytes: 0 }; @@ -1082,11 +932,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } } - // Any non-empty finish_reason ends the generation: flush assembled tool calls as - // atomic sequences (covers "tool_calls" AND providers that close tool turns with "stop"). - if (typeof choice.finish_reason === "string" && choice.finish_reason) { - yield* flushToolCalls(); - } + if (typeof choice.finish_reason === "string" && choice.finish_reason) yield* flushToolCalls(); return "continue"; }; @@ -1125,21 +971,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } } - // Some providers send the terminal `data:` frame (carrying the final delta, finish_reason, - // and/or usage) WITHOUT a trailing newline before closing the socket, so it never crosses - // the split("\n") boundary and stays in `buffer`. Run it through the SAME handler so its - // content/tool-calls are emitted and its terminal signal observed — otherwise a genuinely - // complete stream loses its last frame and may be falsely failed below. if (buffer.length > 0) { if ((yield* handleDataLine(buffer)) === "terminate") return; } - // Reader EOF. Prefer failing closed before flushing pending tool calls so the bridge - // never sees a fabricated tool_call_end on a truncated mid-assembly stream. - // - // Checked BEFORE flushToolCalls(), because that helper emits tool_call_end and there is no - // taking it back: a half-assembled argument string would reach the client as a completed - // call. Tool calls are buffered here (unlike the Anthropic adapter, which forwards - // fragments live), so this adapter can still decide. const sawFinish = finishReason !== undefined; if (!sawFinish && pendingToolCalls.length > 0) { debugProviderDiagnostic("openai-chat", "stream-truncated", { @@ -1150,9 +984,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd yield { type: "error", message: "upstream stream ended mid tool call without a terminal signal — possible truncation" }; return; } - // Finish-less EOF is only safe when answer text was emitted. Reasoning-only / usage-only - // truncations must stay on the error path (hideThinkingSummary can suppress reasoning). - // Trailing usage alone is not a terminal signal for this adapter (#735 / restore #773). if (!sawFinish && !sawUserFacingOutput) { debugProviderDiagnostic("openai-chat", "stream-truncated", { finishReason: finishReason ?? null, @@ -1162,7 +993,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return; } yield* flushToolCalls(); - // Graceful close that omitted [DONE] but delivered finish_reason and/or answer text. const stopReason = stopReasonFor(finishReason); yield { type: "done", usage: pendingUsage, ...(stopReason ? { stopReason } : {}) }; } catch (error) { @@ -1191,60 +1021,54 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const responseBytes = new TextEncoder().encode(JSON.stringify(json)).byteLength; budget.chargeRetained(responseBytes, { kind: "retained_collectors" }); try { - const payload = unwrapChatCompletionPayload(json); - const usage = usageFromOpenAIChat(payload.usage as Record | undefined); - if (json.success === false && payload.error === undefined) { - return [{ - type: "error", - message: "upstream reported failure without an error payload", - ...(usage ? { usage } : {}), - }]; - } - if (payload.error !== undefined && payload.error !== null) { - return [upstreamErrorEvent(payload.error, usage)]; - } - - const events: AdapterEvent[] = []; - const choices = payload.choices as { - message?: Record; - finish_reason?: unknown; - error?: OpenAIChatError; - }[] | undefined; - if (!Array.isArray(choices) || choices.length === 0) { - return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; - } - const rawChoice = choices[0]; - if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { - return [invalidChoicesEvent(usage)]; - } - const choice = rawChoice; - if (choice.finish_reason === "error") return [upstreamErrorEvent(choice.error, usage)]; - if (!choice.message) return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; - - const msg = choice.message; - const reasoningText = reasoningTextFrom(msg); - if (reasoningText !== undefined) { - events.push({ type: "reasoning_raw_delta", text: reasoningText }); - } - if (typeof msg.content === "string") { - events.push({ type: "text_delta", text: msg.content }); - } - const toolCalls = msg.tool_calls as { id: string; function: { name: string; arguments: string } }[] | undefined; - if (toolCalls) { - for (const tc of toolCalls) { - events.push({ type: "tool_call_start", id: tc.id, name: tc.function.name }); - events.push({ type: "tool_call_delta", arguments: tc.function.arguments }); - events.push({ type: "tool_call_end" }); + const payload = unwrapChatCompletionPayload(json); + const usage = usageFromOpenAIChat(payload.usage as Record | undefined); + if (json.success === false && payload.error === undefined) { + return [{ + type: "error", + message: "upstream reported failure without an error payload", + ...(usage ? { usage } : {}), + }]; } - } - const stopReason = stopReasonFor(choice.finish_reason); - events.push({ - type: "done", - usage, - ...(stopReason ? { stopReason } : {}), - }); - retainTranslatedEventBatch(events, budget); - return events; + if (payload.error !== undefined && payload.error !== null) return [upstreamErrorEvent(payload.error, usage)]; + + const events: AdapterEvent[] = []; + const choices = payload.choices as { + message?: Record; + finish_reason?: unknown; + error?: OpenAIChatError; + }[] | undefined; + if (!Array.isArray(choices) || choices.length === 0) { + return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; + } + const rawChoice = choices[0]; + if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { + return [invalidChoicesEvent(usage)]; + } + const choice = rawChoice; + if (choice.finish_reason === "error") return [upstreamErrorEvent(choice.error, usage)]; + if (!choice.message) return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; + + const msg = choice.message; + const reasoningText = reasoningTextFrom(msg); + if (reasoningText !== undefined) events.push({ type: "reasoning_raw_delta", text: reasoningText }); + if (typeof msg.content === "string") events.push({ type: "text_delta", text: msg.content }); + const toolCalls = msg.tool_calls as { id: string; function: { name: string; arguments: string } }[] | undefined; + if (toolCalls) { + for (const tc of toolCalls) { + events.push({ type: "tool_call_start", id: tc.id, name: tc.function.name }); + events.push({ type: "tool_call_delta", arguments: tc.function.arguments }); + events.push({ type: "tool_call_end" }); + } + } + const stopReason = stopReasonFor(choice.finish_reason); + events.push({ + type: "done", + usage, + ...(stopReason ? { stopReason } : {}), + }); + retainTranslatedEventBatch(events, budget); + return events; } finally { budget.releaseRetained(responseBytes, { kind: "retained_collectors" }); } diff --git a/src/lab/conformance/assertion.ts b/src/lab/conformance/assertion.ts index 0f4fd2bb22..adc94fd4f8 100644 --- a/src/lab/conformance/assertion.ts +++ b/src/lab/conformance/assertion.ts @@ -129,7 +129,9 @@ function evaluatePresence( function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { const events = observation.client.response.events.map((e) => e.event); const expected = assertion.expected as string[]; - const passed = events.length === expected.length && events.every((e, i) => e === expected[i]); + const passed = Array.isArray(expected) + && events.length === expected.length + && events.every((e, i) => e === expected[i]); return { id: assertion.id, operator: assertion.operator, @@ -167,7 +169,7 @@ function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObse }; } const grammar = ID_GRAMMARS[String(assertion.expected)]; - const value = String(resolved.value ?? ""); + const value = typeof resolved.value === "string" ? resolved.value : ""; const passed = grammar ? grammar.test(value) : false; return { id: assertion.id, @@ -180,7 +182,17 @@ function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObse } function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const pointers = assertion.expected as string[]; + const pointers = assertion.expected; + if (!Array.isArray(pointers) || pointers.length < 2 || !pointers.every((p) => typeof p === "string")) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected at least two pointers", + reason: "invalid_expected", + }; + } const values: string[] = []; for (const pointer of pointers) { const resolved = resolveJsonPointer(observation, pointer); @@ -194,7 +206,17 @@ function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObser reason: resolved.reason, }; } - values.push(String(resolved.value ?? "")); + if (typeof resolved.value !== "string") { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "identifier must be a string", + reason: "selector_type_mismatch", + }; + } + values.push(resolved.value); } const passed = values.every((v) => v === values[0]); return { @@ -207,9 +229,17 @@ function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObser }; } +function correlatedIds(left: unknown, right: unknown): boolean { + return typeof left === "string" + && typeof right === "string" + && left.length > 0 + && right.length > 0 + && left === right; +} + function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const pointers = assertion.expected as string[]; - if (pointers.length !== 2) { + const pointers = assertion.expected; + if (!Array.isArray(pointers) || pointers.length !== 2 || !pointers.every((p) => typeof p === "string")) { return { id: assertion.id, operator: assertion.operator, @@ -222,7 +252,6 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO const left = resolveJsonPointer(observation, pointers[0]); const right = resolveJsonPointer(observation, pointers[1]); if (!left.ok || !right.ok) { - const reason = !left.ok ? (left as { reason: string }).reason : (right as { reason: string }).reason; return { id: assertion.id, operator: assertion.operator, @@ -232,7 +261,7 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO reason: "selector_missing", }; } - const passed = String(left.value ?? "") === String(right.value ?? ""); + const passed = correlatedIds(left.value, right.value); return { id: assertion.id, operator: assertion.operator, @@ -244,7 +273,17 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO } function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const spec = assertion.expected as { call: string; result: string }; + const spec = assertion.expected as { call?: unknown; result?: unknown }; + if (!spec || typeof spec.call !== "string" || typeof spec.result !== "string") { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected call/result pointers", + reason: "invalid_expected", + }; + } const call = resolveJsonPointer(observation, spec.call); const result = resolveJsonPointer(observation, spec.result); if (!call.ok || !result.ok) { @@ -257,7 +296,7 @@ function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: Nor reason: "selector_missing", }; } - const passed = String(call.value ?? "") === String(result.value ?? ""); + const passed = correlatedIds(call.value, result.value); return { id: assertion.id, operator: assertion.operator, diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 222eb80720..1e3d349d61 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -10,10 +10,10 @@ import { expandPreviousResponseInput, rememberResponseState, } from "../../responses/state"; -import type { AdapterEvent, OcxParsedRequest } from "../../types"; -import { withHarnessTranslatorBudget } from "./harness-budget"; +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../types"; import { evaluateAssertions } from "./assertion"; import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { withHarnessTranslatorBudget } from "./harness-budget"; import { attachMcpVerifiers, executeMcpSyntheticAction } from "./mcp-stub"; import { attachVerifiers, @@ -22,7 +22,7 @@ import { filterAnthropicEvents, recordUpstreamRequest, } from "./observation"; -import { eventsFromBridgeFrames, normalizeSseBytes } from "./sse-normalize"; +import { normalizeSseBytes } from "./sse-normalize"; import type { CaseRecord, NormalizedObservation, ScenarioRunResult } from "./types"; async function collectAdapterEvents(gen: AsyncGenerator): Promise { @@ -32,7 +32,6 @@ async function collectAdapterEvents(gen: AsyncGenerator): Promise< } async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model"): Promise<{ - frames: Array<{ event?: string; data: Record }>; events: ReturnType; }> { async function* replay(): AsyncGenerator { @@ -42,28 +41,32 @@ async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model") const reader = stream.getReader(); const decoder = new TextDecoder(); let text = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - text += decoder.decode(value, { stream: true }); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + text += decoder.decode(); + } finally { + reader.releaseLock(); } - const frames = text.split("\n\n") - .map((frame) => frame.trim()) - .filter((frame) => frame.length > 0 && frame !== "data: [DONE]") - .map((frame) => { - const lines = frame.split("\n"); - const event = lines.find((l) => l.startsWith("event: "))?.slice(7); - const dataLine = lines.find((l) => l.startsWith("data: ")); - return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; - }); - const normalized = eventsFromBridgeFrames(frames); - return { frames, events: normalized }; + // bridgeToResponsesSSE appends a client-transport [DONE] padding frame. It is not an + // upstream OpenAI-Chat sentinel, so remove only that exact bridge-owned trailer before + // feeding the remaining Responses frames to the shared normalizer. + const trailer = "data: [DONE]\n\n"; + const framed = text.endsWith(trailer) ? text.slice(0, -trailer.length) : text; + return { events: normalizeSseBytes(new TextEncoder().encode(framed), "openai-responses") }; } async function parseUpstreamSse(adapter: ReturnType, body: string): Promise { const budget = createTranslatorBudget(); - const response = new Response(body, { headers: { "Content-Type": "text/event-stream" } }); - return await collectAdapterEvents(adapter.parseStream(response, budget)); + try { + const response = new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }); + return await collectAdapterEvents(adapter.parseStream(response, budget)); + } finally { + budget.dispose(); + } } function parsedFromContext(vector: Record): OcxParsedRequest { @@ -71,24 +74,20 @@ function parsedFromContext(vector: Record): OcxParsedRequest { const options = vector.options as Record | undefined; const messages = context?.messages as Array> | undefined; const input = messages - ? messages.map((m) => { - if (m.role === "developer") return { role: "developer", content: m.content }; - return { role: m.role, content: m.content }; - }) + ? messages.map((m) => ({ role: m.role, content: m.content })) : vector.input ?? "PING"; const body: Record = { model: vector.modelId ?? "fixture-model", input, stream: vector.stream ?? false, ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), + ...(options?.reasoning !== undefined ? { reasoning: { effort: options.reasoning } } : {}), ...(options?.textFormat ? { text: { format: options.textFormat } } : {}), ...(vector.tools ? { tools: normalizeTools(vector.tools as unknown[]) } : {}), ...(vector.tool_choice ? { tool_choice: vector.tool_choice } : {}), ...(vector.text ? { text: vector.text } : {}), }; - if (context?.systemPrompt) { - body.instructions = (context.systemPrompt as string[])[0]; - } + if (context?.systemPrompt) body.instructions = (context.systemPrompt as string[])[0]; return parseRequest(body); } @@ -101,6 +100,29 @@ function normalizeTools(tools: unknown[]): unknown[] { }); } +function createHarnessAdapter(provider: OcxProviderConfig) { + return withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); +} + +async function runBuildRequest( + observation: NormalizedObservation, + parsed: OcxParsedRequest, + provider: OcxProviderConfig, +): Promise { + const adapter = createHarnessAdapter(provider); + try { + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + return observation; + } finally { + adapter.dispose(); + } +} + async function executeAdapterVector(caseRecord: CaseRecord): Promise { const observation = emptyObservation(); const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; @@ -112,28 +134,28 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise; const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "openai-responses"); - finalizeObservation(observation, sseEvents, json); + finalizeObservation(observation, sseEvents, json, 200); attachVerifiers(observation, caseRecord); return observation; + } case "chat-core.protocol.request-mapping": return await runBuildRequest(observation, parsedFromContext(vector), provider); - case "anthropic-core.protocol.request-mapping": + case "anthropic-core.protocol.request-mapping": { const anthropicBody = JSON.parse(caseRecord.fixture.bytesUtf8); const translated = anthropicToResponsesTranslation(anthropicBody); - const parsedAnthropic = parseRequest(translated.body); - const responsesProvider = fixtureProviderConfig("openai-responses"); - return await runBuildRequest(observation, parsedAnthropic, responsesProvider); + return await runBuildRequest(observation, parseRequest(translated.body), fixtureProviderConfig("openai-responses")); + } - case "anthropic-core.protocol.tool-round-trip": + case "anthropic-core.protocol.tool-round-trip": { const toolBody = JSON.parse(caseRecord.fixture.bytesUtf8); - const toolTranslated = anthropicToResponsesTranslation(toolBody); - const parsedTool = parseRequest(toolTranslated.body); - return await runBuildRequest(observation, parsedTool, fixtureProviderConfig("openai-responses")); + const translated = anthropicToResponsesTranslation(toolBody); + return await runBuildRequest(observation, parseRequest(translated.body), fixtureProviderConfig("openai-responses")); + } case "tools-core.protocol.function-round-trip": return await runToolRoundTrip(observation, vector, provider); @@ -142,6 +164,7 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise, -): Promise { - const adapter = withHarnessTranslatorBudget( - provider.adapter === "openai-responses" - ? createResponsesPassthroughAdapter(provider) - : createOpenAIChatAdapter(provider), - ); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const json = JSON.parse(built.body); - recordUpstreamRequest(observation, json); - return observation; -} - async function runToolRoundTrip( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const tools = normalizeTools(vector.tools as unknown[]); - const upstreamToolCall = vector.upstreamToolCall as Record; - const toolResult = vector.toolResult as Record; - const parsed1 = parseRequest({ - model: "fixture-model", - input: "PING", - tools, - stream: false, - }); - const built1 = await adapter.buildRequest(parsed1, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built1.body)); - const sseBody = [ - `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, - `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, - "data: [DONE]\n\n", - ].join(""); - const events1 = await parseUpstreamSse(adapter, sseBody); - const bridged = await collectBridgeSse(events1); - finalizeObservation(observation, bridged.events); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, - { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, - ], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + try { + const tools = normalizeTools(vector.tools as unknown[]); + const upstreamToolCall = vector.upstreamToolCall as Record; + const toolResult = vector.toolResult as Record; + const parsed1 = parseRequest({ model: "fixture-model", input: "PING", tools, stream: false }); + const built1 = await adapter.buildRequest(parsed1, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + + const sseBody = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + const events1 = await parseUpstreamSse(adapter, sseBody); + const bridged = await collectBridgeSse(events1); + finalizeObservation(observation, bridged.events, null, 200); + + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, + { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, + ], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runCustomToolRoundTrip( observation: NormalizedObservation, vector: Record, ): Promise { - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const tool = vector.tool as Record; - const call = vector.call as Record; - const output = vector.output as Record; - const parsed1 = parseRequest({ - model: "fixture-model", - input: "PING", - tools: [tool], - stream: false, - }); - const built1 = await adapter.buildRequest(parsed1, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built1.body)); - const events: AdapterEvent[] = [ - { type: "tool_call_start", id: String(call.id), name: String(call.name) }, - { type: "tool_call_delta", arguments: String(call.input) }, - { type: "tool_call_end" }, - { type: "done" }, - ]; - const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, - { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, - ], - tools: [tool], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const tool = vector.tool as Record; + const call = vector.call as Record; + const output = vector.output as Record; + const parsed1 = parseRequest({ model: "fixture-model", input: "PING", tools: [tool], stream: false }); + const built1 = await adapter.buildRequest(parsed1, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + + const bridged = await collectBridgeSse([ + { type: "tool_call_start", id: String(call.id), name: String(call.name) }, + { type: "tool_call_delta", arguments: String(call.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]); + finalizeObservation(observation, bridged.events, null, 200); + + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, + { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, + ], + tools: [tool], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runToolResultContent( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const content = vector.content as Array>; - const parsed = parseRequest({ - model: "fixture-model", - input: [{ type: "function_call_output", call_id: vector.callId, output: content }], - stream: false, - }); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const upstreamJson = normalizeImageToolResultUpstream(JSON.parse(built.body) as Record); - recordUpstreamRequest(observation, upstreamJson); - return observation; + try { + const content = (vector.content ?? vector.result) as Array>; + const parsed = parseRequest({ + model: "fixture-model", + input: [{ type: "function_call_output", call_id: vector.callId, output: content }], + stream: false, + }); + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + recordUpstreamRequest(observation, normalizeImageToolResultUpstream(JSON.parse(built.body) as Record)); + return observation; + } finally { + adapter.dispose(); + } } function normalizeImageToolResultUpstream(body: Record): Record { const messages = body.messages as Array> | undefined; if (!messages) return body; const toolIdx = messages.findIndex((m) => m.role === "tool"); - const userIdx = messages.findIndex((m) => { - if (m.role !== "user" || !Array.isArray(m.content)) return false; - return (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - }); + const userIdx = messages.findIndex((m) => m.role === "user" && Array.isArray(m.content) + && (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url")); if (toolIdx < 0 || userIdx < 0) return body; const tool = messages[toolIdx]; const user = messages[userIdx]; @@ -314,57 +316,51 @@ function normalizeImageToolResultUpstream(body: Record): Record async function runApplyPatchTurn( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const events: AdapterEvent[] = [ - { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, - { type: "tool_call_delta", arguments: String(vector.input) }, - { type: "tool_call_end" }, - { type: "done" }, - ]; - const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events); - recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, - { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, - ], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + try { + const bridged = await collectBridgeSse([ + { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, + { type: "tool_call_delta", arguments: String(vector.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]); + finalizeObservation(observation, bridged.events, null, 200); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, + { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, + ], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runCodexToolContinuation( observation: NormalizedObservation, vector: Record, ): Promise { - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const turn1 = vector.turn1 as { output: unknown[] }; - const turn2 = vector.turn2 as { input: unknown[] }; - const parsed = parseRequest({ - model: "fixture-model", - input: turn2.input, - stream: false, - }); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; - if (Array.isArray(turn1.output)) { - upstreamJson.input = [...turn1.output, ...(upstreamJson.input as unknown[] ?? [])]; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const turn1 = vector.turn1 as { output: unknown[] }; + const turn2 = vector.turn2 as { input: unknown[] }; + const parsed = parseRequest({ model: "fixture-model", input: turn2.input, stream: false }); + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; + if (Array.isArray(turn1.output)) upstreamJson.input = [...turn1.output, ...(upstreamJson.input ?? [])]; + recordUpstreamRequest(observation, upstreamJson); + return observation; + } finally { + adapter.dispose(); } - recordUpstreamRequest(observation, upstreamJson); - return observation; } async function runPreviousResponseReplay( @@ -372,33 +368,115 @@ async function runPreviousResponseReplay( vector: Record, ): Promise { clearResponseStateForTests(); - const stored = vector.stored as Record; - const next = vector.next as Record; - rememberResponseState( - { input: stored.input, store: true }, - { id: String(stored.id), output: stored.output, status: "completed" }, - undefined, - { force: true }, - ); - const requestBody = { - model: "fixture-model", - store: true, - previous_response_id: stored.id, - input: next.input, + try { + const stored = vector.stored as Record; + const next = vector.next as Record; + rememberResponseState( + { input: stored.input, store: true }, + { id: String(stored.id), output: stored.output, status: "completed" }, + undefined, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + model: "fixture-model", + store: true, + previous_response_id: stored.id, + input: next.input, + }); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const built = await adapter.buildRequest({ ...parseRequest(expanded), _previousResponseInputExpanded: true }, { headers: new Headers() }); + const upstreamJson = JSON.parse(built.body) as Record; + delete upstreamJson.previous_response_id; + recordUpstreamRequest(observation, upstreamJson); + return observation; + } finally { + adapter.dispose(); + } + } finally { + clearResponseStateForTests(); + } +} + +async function runReasoningEffortMapping( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider: OcxProviderConfig = { + ...fixtureProviderConfig("openai-chat"), + reasoningEffortMap: vector.reasoningEffortMap as Record, + reasoningWireFormat: vector.reasoningWireFormat as OcxProviderConfig["reasoningWireFormat"], }; - const expanded = expandPreviousResponseInput(requestBody); - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const parsed = parseRequest(expanded); - const built = await adapter.buildRequest({ ...parsed, _previousResponseInputExpanded: true }, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), + const parsed = parseRequest({ + model: "fixture-model", + input: "PING", + stream: false, + reasoning: { effort: vector.requested }, }); - const upstreamJson = JSON.parse(built.body) as Record; - delete upstreamJson.previous_response_id; - recordUpstreamRequest(observation, upstreamJson); + return await runBuildRequest(observation, parsed, provider); +} + +async function runReasoningReplay( + observation: NormalizedObservation, + vector: Record, +): Promise { + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const turn1 = vector.turn1 as { + reasoning: { id: string; text: string; signature: string }; + toolCall: { callId: string }; + }; + const turn2 = vector.turn2 as { toolResult: { callId: string; output: string } }; + const first = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: "PING", stream: false }), { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(first.body)); + const replayInput = [ + { + type: "reasoning", + id: turn1.reasoning.id, + content: [{ type: "reasoning_text", text: turn1.reasoning.text }], + signature: turn1.reasoning.signature, + }, + { + type: "function_call_output", + call_id: turn2.toolResult.callId, + output: turn2.toolResult.output, + }, + ]; + const second = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: replayInput, stream: false }), { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(second.body)); + return observation; + } finally { + adapter.dispose(); + } +} + +async function runReasoningPrivateIsolation( + observation: NormalizedObservation, + vector: Record, +): Promise { clearResponseStateForTests(); - return observation; + try { + const origin = vector.origin as { encrypted?: string }; + rememberResponseState( + { input: "PING", store: true }, + { + id: "resp_private_fixture", + output: [{ type: "reasoning", id: "rs_private", summary: [], encrypted_content: origin.encrypted }], + status: "completed", + }, + undefined, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + model: "fixture-model", + store: true, + previous_response_id: "resp_private_fixture", + input: "NEXT", + }); + return await runBuildRequest(observation, parseRequest(expanded), fixtureProviderConfig("openai-chat")); + } finally { + clearResponseStateForTests(); + } } async function executeClientRequest(caseRecord: CaseRecord): Promise { @@ -409,20 +487,20 @@ async function executeClientRequest(caseRecord: CaseRecord): Promise { + if (!caseRecord.initiatingRequest) return; + const body = JSON.parse(caseRecord.initiatingRequest.bytesUtf8); + const inbound = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const parsed = inbound === "anthropic-messages" + ? parseRequest(anthropicToResponsesTranslation(body).body) + : parseRequest(body); + const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + await runBuildRequest(observation, parsed, fixtureProviderConfig(upstreamAdapterForProtocol(upstream))); } async function executeStreamScenario(caseRecord: CaseRecord): Promise { @@ -431,88 +509,94 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise; let json: Record | null = null; + let responseStatus = 200; if (upstreamProtocol === "openai-chat") { - const provider = fixtureProviderConfig("openai-chat"); - const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); - const bridged = await collectBridgeSse(adapterEvents); - events = bridged.events; - if (surface.includes("anthropic")) { - const budget = createTranslatorBudget(); - const bridgedStream = bridgeToResponsesSSE((async function* () { - for (const event of adapterEvents) yield event; - })(), "fixture-model"); - const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); - const reader = anthropicStream.getReader(); - const decoder = new TextDecoder(); - let anthropicText = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - anthropicText += decoder.decode(value, { stream: true }); - } - events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(anthropicText), "anthropic-messages")); - } - if (caseRecord.id === "codex-core.protocol.streaming-turn" && events.length > 0) { - const data = events[0].data; - if (data && typeof data === "object") { - (data as Record).phase = "final_answer"; + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); + try { + const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); + events = (await collectBridgeSse(adapterEvents)).events; + if (surface.includes("anthropic")) { + const budget = createTranslatorBudget(); + try { + const bridgedStream = bridgeToResponsesSSE((async function* () { + for (const event of adapterEvents) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + } finally { + reader.releaseLock(); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); + } finally { + budget.dispose(); + } } + } finally { + adapter.dispose(); } } else if (upstreamProtocol === "openai-responses") { if (inboundProtocol === "anthropic-messages") { const budget = createTranslatorBudget(); - const responsesSse = bridgeToResponsesSSE((async function* () { - const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); - const budgetInner = createTranslatorBudget(); - const response = new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "text/event-stream" } }); - for await (const event of passthrough.parseStream(response, budgetInner)) yield event; - })(), "fixture-model"); - const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); - const reader = anthropicStream.getReader(); - const decoder = new TextDecoder(); - let text = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - text += decoder.decode(value, { stream: true }); - } - events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); - if (caseRecord.id === "anthropic-core.protocol.terminal-errors") { - events = events.filter((e) => e.event === "error"); + const passthroughBudget = createTranslatorBudget(); + try { + const responsesSse = bridgeToResponsesSSE((async function* () { + const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); + const response = new Response(caseRecord.fixture.bytesUtf8, { status: 200, headers: { "Content-Type": "text/event-stream" } }); + for await (const event of passthrough.parseStream(response, passthroughBudget)) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + } finally { + reader.releaseLock(); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); + } finally { + passthroughBudget.dispose(); + budget.dispose(); } } else { events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } } else { - events = normalizeSseBytes(upstreamBytes, upstreamProtocol); + throw new Error(`unsupported upstream protocol: ${upstreamProtocol}`); } if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { - const provider = fixtureProviderConfig("openai-chat"); - const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); - const parsedEvents = adapter.parseResponse - ? await adapter.parseResponse( - new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "application/json" } }), - createTranslatorBudget(), - ) - : []; - const bridged = await collectBridgeSse(parsedEvents); - events = bridged.events; - json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; - finalizeObservation(observation, events, json); - return observation; + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); + try { + const response = new Response(caseRecord.fixture.bytesUtf8, { status: 200, headers: { "Content-Type": "application/json" } }); + responseStatus = response.status; + const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); + const parsedEvents = adapter.parseResponse ? await adapter.parseResponse(response) : []; + events = (await collectBridgeSse(parsedEvents)).events; + json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; + } finally { + adapter.dispose(); + } } - finalizeObservation(observation, events, json); + finalizeObservation(observation, events, json, responseStatus); attachVerifiers(observation, caseRecord); return observation; } @@ -530,7 +614,9 @@ export async function executeScenario(caseRecord: CaseRecord): Promise assertionResults.find((r) => r.id === id)?.passed); - const expectedFailureMatched = controlPassed - && requiredFailures.length === 0; + const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed === true); + const expectedFailureMatched = controlPassed && requiredFailures.length === 0; return { scenarioId: caseRecord.id, suite: caseRecord.suite, diff --git a/src/lab/conformance/fixture-provider.ts b/src/lab/conformance/fixture-provider.ts index 1ef49185f9..d389deb673 100644 --- a/src/lab/conformance/fixture-provider.ts +++ b/src/lab/conformance/fixture-provider.ts @@ -3,7 +3,10 @@ import type { OcxProviderConfig } from "../../types"; export function fixtureProviderConfig(adapter: string): OcxProviderConfig { return { adapter, - baseUrl: "http://127.0.0.1:1/v1", + // The Chat fixture intentionally exercises native OpenAI Chat semantics (including + // role:"developer" and named single-tool selection). Other fixture adapters remain + // loopback-only and never perform network I/O. + baseUrl: adapter === "openai-chat" ? "https://api.openai.com/v1" : "http://127.0.0.1:1/v1", apiKey: "fixture-key", allowPrivateNetwork: true, models: ["fixture-model"], @@ -18,11 +21,7 @@ export function upstreamAdapterForProtocol(protocol: string): string { return "openai-chat"; case "openai-responses": return "openai-responses"; - case "anthropic-messages": - return "anthropic"; - case "cursor-protobuf": - return "cursor"; default: - return "openai-chat"; + throw new Error(`unsupported upstream protocol: ${protocol}`); } } diff --git a/src/lab/conformance/harness-budget.ts b/src/lab/conformance/harness-budget.ts index b227af3bfc..a161f5ec2e 100644 --- a/src/lab/conformance/harness-budget.ts +++ b/src/lab/conformance/harness-budget.ts @@ -11,14 +11,16 @@ type TestAdapter = Omit ReturnType>; + dispose(): void; }; -/** Inject translator budget for harness adapter calls (mirrors tests/helpers/translator-budget). */ +/** Inject one bounded translator budget for a harness adapter scope. Call dispose() in finally. */ export function withHarnessTranslatorBudget(adapter: T): TestAdapter { const budget = createTranslatorBudget(); const buildRequest = adapter.buildRequest.bind(adapter); const parseStream = adapter.parseStream.bind(adapter); const parseResponse = adapter.parseResponse?.bind(adapter); + let disposed = false; return { ...adapter, buildRequest(parsed: Parameters[0], incoming?: Partial) { @@ -36,5 +38,10 @@ export function withHarnessTranslatorBudget(adapter: return parseResponse(response, explicitBudget ?? budget); }, } : {}), + dispose() { + if (disposed) return; + disposed = true; + budget.dispose(); + }, } as unknown as TestAdapter; } diff --git a/src/lab/conformance/jcs.ts b/src/lab/conformance/jcs.ts index 4fa996107e..6bbcb923c7 100644 --- a/src/lab/conformance/jcs.ts +++ b/src/lab/conformance/jcs.ts @@ -1,7 +1,10 @@ /** RFC 8785 JSON Canonicalization Scheme (JCS) for deterministic equality. */ export function jcsStringify(value: unknown): string { - if (value === null || typeof value === "boolean" || typeof value === "number") { + if (value === undefined) throw new TypeError("jcsStringify: undefined is not representable in JCS"); + if (value === null || typeof value === "boolean") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("jcsStringify: non-finite numbers are not representable in JCS"); return JSON.stringify(value); } if (typeof value === "string") return JSON.stringify(value); @@ -13,7 +16,7 @@ export function jcsStringify(value: unknown): string { const keys = Object.keys(obj).sort(); return `{${keys.map((k) => `${JSON.stringify(k)}:${jcsStringify(obj[k])}`).join(",")}}`; } - return JSON.stringify(value); + throw new TypeError(`jcsStringify: unsupported value type ${typeof value}`); } export function jcsEqual(a: unknown, b: unknown): boolean { diff --git a/src/lab/conformance/json-pointer.ts b/src/lab/conformance/json-pointer.ts index efc59b7120..1676d0abf1 100644 --- a/src/lab/conformance/json-pointer.ts +++ b/src/lab/conformance/json-pointer.ts @@ -26,7 +26,9 @@ export function resolveJsonPointer(root: unknown, pointer: string): PointerResul return { ok: false, reason: "selector_missing" }; } const obj = current as Record; - if (!(token in obj)) return { ok: false, reason: "selector_missing" }; + if (!Object.prototype.hasOwnProperty.call(obj, token)) { + return { ok: false, reason: "selector_missing" }; + } current = obj[token]; } return { ok: true, value: current }; diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts index a44fb4bc12..df8001239b 100644 --- a/src/lab/conformance/manifest.ts +++ b/src/lab/conformance/manifest.ts @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import { fixtureDigest } from "./digest"; import { MCP_ACTION_TOKENS } from "./mcp-stub"; import type { CaseAuthority, @@ -12,6 +12,7 @@ import type { import { CL01_SUITES, SYNTHETIC_MARKER } from "./types"; const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +// Provenance is the normative CL-00 authority name, not the runtime copy's basename. const AUTHORITY_FILE = "022_protocol_v1_cases.json"; export function loadCaseAuthority(): CaseAuthority { @@ -74,7 +75,12 @@ function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): R } function expandFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { - const base = [...authority.failureRuleSets[authority.manifestDefaults.failureRuleSet]]; + const setName = authority.manifestDefaults.failureRuleSet; + const ruleSet = authority.failureRuleSets[setName]; + if (!Array.isArray(ruleSet)) { + throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); + } + const base = [...ruleSet]; if (!caseRecord.expectedFailure) return base; const template = authority.expectedFailureRuleTemplate; const controlRule: FailureRule = { @@ -133,12 +139,6 @@ export function validateExpandedFixtureRef( return errors; } -export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { - const expanded = expandScenario(caseRecord, authority); - const digest = scenarioManifestDigest(expanded); - return digest.length === 64; -} - function validateMcpHarnessFeatures(caseRecord: CaseRecord): string[] { if (caseRecord.suite !== "mcp-core") return []; const tokens = caseRecord.requirements.requiredHarnessFeatures.filter( diff --git a/src/lab/conformance/mcp-stub.ts b/src/lab/conformance/mcp-stub.ts index bf04e9cff1..ee50b3599a 100644 --- a/src/lab/conformance/mcp-stub.ts +++ b/src/lab/conformance/mcp-stub.ts @@ -30,14 +30,19 @@ export function executeMcpSyntheticAction(caseRecord: CaseRecord): NormalizedObs return runCallResult(decoded); case "mcp_resource_round_trip_v1": return runResourceRoundTrip(decoded); - default: - throw new Error(`invalid_manifest: unsupported MCP action ${token}`); } } +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + function runNamespaceRoundTrip(decoded: Record): NormalizedObservation { - const namespace = String(decoded.namespace ?? ""); - const name = String(decoded.name ?? ""); + if (!nonEmptyString(decoded.namespace) || !nonEmptyString(decoded.name)) { + throw new Error("invalid_manifest: MCP namespace/name must be non-empty strings"); + } + const namespace = decoded.namespace; + const name = decoded.name; const wireName = `${namespace}__${name}`; const observation = emptyObservation(); observation.upstream.requests.push({ @@ -72,29 +77,50 @@ function utf8ByteLength(value: string): number { return new TextEncoder().encode(value).byteLength; } +function parsesJson(value: string): boolean { + try { + JSON.parse(value); + return true; + } catch { + return false; + } +} + function runSchemaBounds(decoded: Record): NormalizedObservation { - const limitBytes = Number(decoded.limitBytes ?? 0); - const exactSchema = String(decoded.exactSchema ?? ""); - const overSchema = String(decoded.overSchema ?? ""); const observation = emptyObservation(); - const exactBound = utf8ByteLength(exactSchema) === limitBytes && JSON.parse(exactSchema) !== undefined - ? "pass" - : "fail"; - const oneOverRejected = utf8ByteLength(overSchema) === limitBytes + 1 - && JSON.parse(overSchema) !== undefined - ? "pass" - : "fail"; + const limitBytes = decoded.limitBytes; + const exactSchema = decoded.exactSchema; + const overSchema = decoded.overSchema; + if (!Number.isInteger(limitBytes) || (limitBytes as number) <= 0 + || typeof exactSchema !== "string" || typeof overSchema !== "string") { + observation.verifiers = { + exact_bound: "fail", + one_over_rejected: "fail", + partial_commit: false, + }; + return observation; + } + + const limit = limitBytes as number; + const exactValid = utf8ByteLength(exactSchema) === limit && parsesJson(exactSchema); + // The inert stub models two isolated catalogue transactions. The over-bound transaction + // is rejected before commit; its validity matters so this tests the byte ceiling rather + // than malformed JSON. + const overRejected = utf8ByteLength(overSchema) === limit + 1 && parsesJson(overSchema); observation.verifiers = { - exact_bound: exactBound, - one_over_rejected: oneOverRejected, + exact_bound: exactValid ? "pass" : "fail", + one_over_rejected: overRejected ? "pass" : "fail", partial_commit: false, }; return observation; } function runCallResult(decoded: Record): NormalizedObservation { - const namespace = String(decoded.namespace ?? ""); - const name = String(decoded.name ?? ""); + if (!nonEmptyString(decoded.namespace) || !nonEmptyString(decoded.name)) { + throw new Error("invalid_manifest: MCP namespace/name must be non-empty strings"); + } + const namespace = decoded.namespace; + const name = decoded.name; const argumentsValue = decoded.arguments ?? {}; const result = decoded.result; const wireName = `${namespace}__${name}`; @@ -119,13 +145,22 @@ function runCallResult(decoded: Record): NormalizedObservation } function runResourceRoundTrip(decoded: Record): NormalizedObservation { - const resources = decoded.resources; - const read = decoded.read as { uri?: string; contents?: unknown[] } | undefined; + if (!Array.isArray(decoded.resources) || !decoded.read || typeof decoded.read !== "object") { + throw new Error("invalid_manifest: invalid MCP resource fixture"); + } + const read = decoded.read as { uri?: unknown; contents?: unknown[] }; + if (!nonEmptyString(read.uri) || !Array.isArray(read.contents)) { + throw new Error("invalid_manifest: invalid MCP resource read fixture"); + } + const matching = decoded.resources.filter((resource) => + resource && typeof resource === "object" && (resource as { uri?: unknown }).uri === read.uri + ); + if (matching.length !== 1) throw new Error("invalid_manifest: MCP resource URI must resolve exactly once"); const observation = emptyObservation(); setClientResponse(observation, { json: { - resources, - contents: read?.contents, + resources: decoded.resources, + contents: read.contents, }, status: 200, }); @@ -139,12 +174,6 @@ export function attachMcpVerifiers(observation: NormalizedObservation, caseRecor observation.client.response.mcpCalls = projectMcpCalls(toolCalls); } } - if (caseRecord.id === "mcp-core.protocol.call-result") { - const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; - observation.verifiers.stub_received = { - namespace: String(decoded.namespace ?? ""), - name: String(decoded.name ?? ""), - arguments: decoded.arguments ?? {}, - }; - } + // runCallResult records the literal one-invocation receipt. Do not reconstruct it here: + // assertions must inspect the action result that actually ran. } diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts index 04f35a7aed..66ab70f4e7 100644 --- a/src/lab/conformance/negative-controls.ts +++ b/src/lab/conformance/negative-controls.ts @@ -54,7 +54,7 @@ export const NEGATIVE_CONTROL_FIXTURES: Array<{ mutate: (c) => ({ ...c, id: "negative.tool-result-order", - assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/input/0/call_id" }, required: true }], + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/1/tool_call_id" }, required: true }], fixture: { ...c.fixture, bytesUtf8: JSON.stringify({ @@ -152,7 +152,9 @@ export function buildNegativeControls(cases: CaseRecord[]): CaseRecord[] { for (const control of NEGATIVE_CONTROL_FIXTURES) { const base = baseCaseForNegativeControl(control.id, cases); if (!base) continue; - built.push(control.mutate(structuredClone(base))); + const cloned = structuredClone(base); + delete cloned.expectedFailure; + built.push(control.mutate(cloned)); } return built; } diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index cb37317344..287467802f 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -64,24 +64,27 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio if (!item || typeof item !== "object") continue; const rec = item as Record; if (rec.type === "function_call") { - calls.push({ - id: String(rec.call_id ?? rec.id ?? ""), - name: String(rec.name ?? ""), - arguments: parseToolArguments(rec.arguments, "function"), - kind: "function", - ordinal: ordinal++, - }); + const id = rec.call_id ?? rec.id; + const name = rec.name; + if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; + const args = parseToolArguments(rec.arguments, "function"); + if (args === null) continue; + calls.push({ id, name, arguments: args, kind: "function", ordinal: ordinal++ }); } else if (rec.type === "custom_tool_call") { + const id = rec.call_id ?? rec.id; + const name = rec.name; + if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; calls.push({ - id: String(rec.call_id ?? rec.id ?? ""), - name: String(rec.name ?? ""), + id, + name, arguments: parseToolArguments(rec.input, "custom"), kind: "custom", ordinal: ordinal++, }); } } - return calls; + const ids = calls.map((call) => call.id); + return new Set(ids).size === ids.length ? calls : []; } export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { @@ -124,24 +127,28 @@ function deriveTerminal(events: NormalizedEvent[]): string | null { return null; } -export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { - if (json && typeof json === "object" && !Array.isArray(json)) { - const resp = json as Record; - if (Array.isArray(resp.output)) { - let text = ""; - for (const item of resp.output) { - if (!item || typeof item !== "object") continue; - const content = (item as { content?: unknown }).content; - if (!Array.isArray(content)) continue; - for (const part of content) { - if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { - text += String((part as { text?: string }).text ?? ""); - } - } +function extractOutputText(json: Record): string { + let text = ""; + const output = json.output; + if (!Array.isArray(output)) return text; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown[] }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); } - if (text) return text; } } + return text; +} + +export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { + if (json && typeof json === "object" && !Array.isArray(json)) { + const text = extractOutputText(json as Record); + if (text) return text; + } let text = ""; for (const ev of events) { if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { @@ -159,26 +166,30 @@ export function finalizeObservation( observation: NormalizedObservation, events: NormalizedEvent[], json: unknown = null, + status = 200, ): void { - const toolCalls = projectToolCallsFromEvents(events); - const terminal = deriveTerminal(events); - setClientResponse(observation, { - events, - toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( + const eventToolCalls = projectToolCallsFromEvents(events); + const resolvedToolCalls = eventToolCalls.length > 0 + ? eventToolCalls + : projectToolCallsFromOutput( json && typeof json === "object" && !Array.isArray(json) ? ((json as { output?: unknown[] }).output ?? []) : [], - ), - mcpCalls: projectMcpCalls(toolCalls), + ); + const terminal = deriveTerminal(events); + setClientResponse(observation, { + events, + toolCalls: resolvedToolCalls, + mcpCalls: projectMcpCalls(resolvedToolCalls), terminal, normalizedText: deriveNormalizedText(events, json), json, - status: 200, + status, }); } export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { - observation.verifiers = buildVerifiers(observation, caseRecord); + observation.verifiers = { ...observation.verifiers, ...buildVerifiers(observation, caseRecord) }; } function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): Record { @@ -218,23 +229,28 @@ function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseReco } function evaluateCallResultOrder(observation: NormalizedObservation): string { - const input = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; - if (!input?.input || !Array.isArray(input.input)) return "fail"; - let sawCall = false; - for (const item of input.input) { + const request = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; + const input = request?.input; + if (!Array.isArray(input)) return "fail"; + let pendingCallId: string | undefined; + let resultCount = 0; + for (const item of input) { if (!item || typeof item !== "object") continue; - const type = (item as { type?: string }).type; - if (type === "function_call") { - if (sawCall) return "fail"; - sawCall = true; + const record = item as { type?: string; call_id?: unknown }; + if (record.type === "function_call") { + if (pendingCallId !== undefined) return "fail"; + if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; + pendingCallId = record.call_id; continue; } - if (type === "function_call_output") { - if (!sawCall) return "fail"; - return "pass"; + if (record.type === "function_call_output") { + if (pendingCallId === undefined) return "fail"; + if (typeof record.call_id !== "string" || record.call_id !== pendingCallId) return "fail"; + resultCount++; + if (resultCount > 1) return "fail"; } } - return "fail"; + return pendingCallId !== undefined && resultCount === 1 ? "pass" : "fail"; } function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { @@ -287,13 +303,10 @@ function evaluateModalityPath(caseRecord: CaseRecord): string { function evaluateSilentImageDrop(caseRecord: CaseRecord): boolean { const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; - const requestHasImage = Boolean(vector.requestHasImage); - const modalities = vector.modelInputModalities as string[] | undefined; - const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; - if (!requestHasImage) return false; - if (Array.isArray(modalities) && modalities.includes("image")) return false; - if (sidecar?.enabled) return false; - return true; + if (!Boolean(vector.requestHasImage)) return false; + // Protocol V1's closed modality-gate vector treats the explicit `unsupported` path as a + // typed rejection, not as an omitted image. Native/sidecar paths likewise preserve it. + return !["native", "sidecar", "unsupported"].includes(evaluateModalityPath(caseRecord)); } function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { @@ -320,20 +333,3 @@ function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { const sseProjection = { text: sseText, terminal: sseTerminal }; return JSON.stringify(jsonProjection) === JSON.stringify(sseProjection) ? "pass" : "fail"; } - -function extractOutputText(json: Record): string { - let text = ""; - const output = json.output; - if (!Array.isArray(output)) return text; - for (const item of output) { - if (!item || typeof item !== "object") continue; - const content = (item as { content?: unknown[] }).content; - if (!Array.isArray(content)) continue; - for (const part of content) { - if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { - text += String((part as { text?: string }).text ?? ""); - } - } - } - return text; -} diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts index d50057d3ef..864f887266 100644 --- a/src/lab/conformance/runner.ts +++ b/src/lab/conformance/runner.ts @@ -5,17 +5,26 @@ import type { ScenarioRunResult } from "./types"; import { CL01_SUITES } from "./types"; export interface ConformanceRunSummary { + /** Number of scenarios executed. */ total: number; + /** Number of scenarios that met their expected outcome. */ passed: number; + /** Number of scenarios that did not meet their expected outcome. */ failed: number; results: ScenarioRunResult[]; } +export interface NegativeControlRunSummary extends ConformanceRunSummary { + /** Number of deliberately defective controls correctly rejected by the harness. */ + rejected: number; +} + export async function runConformanceSuite( suites: readonly string[] = CL01_SUITES, ): Promise { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, suites); + if (scenarios.length === 0) throw new Error("harness_failure: no CL-01 scenarios discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { results.push(await runScenario(scenario)); @@ -24,15 +33,22 @@ export async function runConformanceSuite( return { total: results.length, passed, failed: results.length - passed, results }; } -export async function runNegativeControls(): Promise { +export async function runNegativeControls(): Promise { const authority = loadCaseAuthority(); const scenarios = buildNegativeControls(discoverScenarios(authority)); + if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { results.push(await runScenario(scenario)); } - const passed = results.filter((r) => !r.passed).length; - return { total: results.length, passed, failed: results.length - passed, results }; + const rejected = results.filter((r) => !r.passed).length; + return { + total: results.length, + passed: rejected, + rejected, + failed: results.length - rejected, + results, + }; } export function listScenarioIds(suites: readonly string[] = CL01_SUITES): string[] { diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 3ce474f821..f1c61bce49 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -152,10 +152,14 @@ export interface ScenarioRunResult { diagnostics: string[]; } +/** All eight protocol-conformance suites frozen by CL-00 Protocol V1. */ export const CL01_SUITES = [ "responses-core", "chat-core", "anthropic-core", "tools-core", "codex-core", + "vision-core", + "reasoning-core", + "mcp-core", ] as const; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts index b0676748e7..a8208da98d 100644 --- a/tests/lab-conformance-harness.test.ts +++ b/tests/lab-conformance-harness.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { evaluateAssertion } from "../src/lab/conformance/assertion"; import { fixtureDigest, scenarioManifestDigest } from "../src/lab/conformance/digest"; -import { jcsEqual } from "../src/lab/conformance/jcs"; +import { jcsEqual, jcsStringify } from "../src/lab/conformance/jcs"; import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; import { runScenario } from "../src/lab/conformance/executor"; import { @@ -10,9 +10,7 @@ import { loadCaseAuthority, validateExpandedFixtureRef, validateFixtureDigests, - validateScenarioManifestDigest, } from "../src/lab/conformance/manifest"; -import { executeMcpSyntheticAction } from "../src/lab/conformance/mcp-stub"; import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; import { emptyObservation } from "../src/lab/conformance/observation"; import { @@ -24,32 +22,59 @@ import { CL01_SUITES, SYNTHETIC_MARKER } from "../src/lab/conformance/types"; import { normalizeSseBytes } from "../src/lab/conformance/sse-normalize"; describe("CL-01 conformance harness infrastructure", () => { - test("loads case authority and validates fixture digests", () => { + test("loads the frozen 35-case authority and validates fixture digests", () => { const authority = loadCaseAuthority(); - expect(authority.cases.length).toBeGreaterThanOrEqual(24); + expect(authority.cases.length).toBe(35); for (const caseRecord of authority.cases) { expect(validateFixtureDigests(caseRecord)).toEqual([]); - expect(validateScenarioManifestDigest(caseRecord, authority)).toBe(true); } }); - test("discovers CL-01 suite scenarios with stable IDs", () => { + test("discovers all eight CL-01 protocol suites with stable IDs", () => { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, CL01_SUITES); - expect(scenarios.length).toBe(24); + expect(scenarios.length).toBe(35); const ids = scenarios.map((s) => s.id); expect(new Set(ids).size).toBe(ids.length); expect(ids).toContain("responses-core.protocol.request-shape"); - expect(ids).toContain("codex-core.protocol.compaction-and-special-items"); + expect(ids).toContain("vision-core.protocol.input-image"); + expect(ids).toContain("reasoning-core.protocol.effort-mapping"); + expect(ids).toContain("mcp-core.protocol.namespace-mapping"); }); - test("json pointer and JCS equality are deterministic", () => { + test("json pointer and JCS equality are deterministic and fail closed", () => { const observation = emptyObservation(); observation.client.response.status = 200; const resolved = resolveJsonPointer(observation, "/client/response/status"); expect(resolved.ok).toBe(true); - expect(jcsEqual(resolved.value, 200)).toBe(true); + expect(resolved.ok && jcsEqual(resolved.value, 200)).toBe(true); expect(jcsEqual({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true); + expect(resolveJsonPointer({}, "/constructor").ok).toBe(false); + expect(() => jcsStringify(undefined)).toThrow(); + expect(() => jcsStringify(Number.NaN)).toThrow(); + }); + + test("identifier operators reject vacuous and null correlations", () => { + const observation = emptyObservation(); + const vacuous = evaluateAssertion({ + id: "stable", + operator: "id_stable_across_events", + selector: "/client", + expected: ["/client/response/status"], + required: true, + }, observation); + expect(vacuous.passed).toBe(false); + expect(vacuous.reason).toBe("invalid_expected"); + + observation.client.response.json = { left: null, right: null }; + const nullCorrelation = evaluateAssertion({ + id: "correlation", + operator: "id_correlates", + selector: "/client/response/json", + expected: ["/client/response/json/left", "/client/response/json/right"], + required: true, + }, observation); + expect(nullCorrelation.passed).toBe(false); }); test("fixture digest matches contract domain separation", () => { @@ -71,7 +96,7 @@ describe("CL-01 conformance harness infrastructure", () => { expect(result.reason).toBe("selector_missing"); }); - test("expanded scenario manifests include synthetic provenance", () => { + test("expanded scenario manifests include normative synthetic provenance", () => { const authority = loadCaseAuthority(); const scenario = discoverScenarios(authority)[0]; const expanded = expandScenario(scenario, authority); @@ -80,8 +105,7 @@ describe("CL-01 conformance harness infrastructure", () => { expect((fixtures[0].provenance as { kind: string }).kind).toBe("lab_authored"); expect((fixtures[0].provenance as { authority: string }).authority).toBe("022_protocol_v1_cases.json"); expect((fixtures[0].provenance as { sourceCommit: string }).sourceCommit).toBe(authority.sourceCommit); - const digest = scenarioManifestDigest(expanded); - expect(digest).toHaveLength(64); + expect(scenarioManifestDigest(expanded)).toHaveLength(64); }); test("rejects forged synthetic provenance metadata", () => { @@ -90,8 +114,8 @@ describe("CL-01 conformance harness infrastructure", () => { const expanded = expandScenario(scenario, authority); const fixtures = expanded.fixtures as Array>; const forged = { ...fixtures[0], syntheticMarker: "forged" }; - const errors = validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8); - expect(errors.some((e) => e.includes("syntheticMarker"))).toBe(true); + expect(validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8) + .some((e) => e.includes("syntheticMarker"))).toBe(true); const badCommit = { ...fixtures[0], provenance: { ...(fixtures[0].provenance as object), sourceCommit: "deadbeef" }, @@ -110,18 +134,35 @@ describe("CL-01 SSE normalization", () => { const anthropicEvents = normalizeSseBytes(bytes, "anthropic-messages"); expect(anthropicEvents.some((e) => e.event === "[DONE]")).toBe(false); }); + + test("normalizes BOM/CRLF, comments, multiline data, and malformed JSON", () => { + const bytes = new TextEncoder().encode( + "\uFEFF: keep-alive\r\nevent: response.completed\r\ndata: {\"type\":\"response.completed\",\r\ndata: \"response\":{\"status\":\"completed\"}}\r\n\r\n" + + "data: {not-json}\r\n\r\n", + ); + const events = normalizeSseBytes(bytes, "openai-responses"); + expect(events).toHaveLength(2); + expect(events[0].event).toBe("response.completed"); + expect(events[0].ordinal).toBe(0); + expect(events[1].event).toBe("malformed"); + expect(events[1].ordinal).toBe(1); + }); + + test("drops scalar/null/array data frames as Protocol V1 padding", () => { + const bytes = new TextEncoder().encode("data: null\n\ndata: 1\n\ndata: []\n\n"); + expect(normalizeSseBytes(bytes, "openai-responses")).toEqual([]); + }); }); describe("CL-01 MCP deterministic actions", () => { - test("all four MCP protocol scenarios pass closed action semantics", async () => { + test("all four MCP scenarios pass through the full runner path", async () => { const authority = loadCaseAuthority(); const mcpScenarios = authority.cases.filter((c) => c.suite === "mcp-core"); expect(mcpScenarios.length).toBe(4); for (const scenario of mcpScenarios) { - const observation = executeMcpSyntheticAction(scenario); - for (const assertion of scenario.assertions) { - const result = evaluateAssertion(assertion, observation); - expect(result.passed).toBe(true); + const result = await runScenario(scenario); + if (!result.passed) { + throw new Error(`${scenario.id}: ${result.diagnostics.join(";")} ${result.assertionResults.filter((a) => !a.passed).map((a) => `${a.id}:${a.reason ?? ""}`).join(",")}`); } } }); @@ -134,18 +175,20 @@ describe("CL-01 expanded scenario manifests are stable", () => { const a = expandScenario(scenario, authority); const b = expandScenario(scenario, authority); expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + expect(scenarioManifestDigest(a)).toBe(scenarioManifestDigest(b)); }); }); describe("CL-01 canonical protocol scenarios", () => { - test("all CL-01 suite scenarios pass", async () => { + test("all 35 CL-01 protocol scenarios pass", async () => { const summary = await runConformanceSuite(); const failures = summary.results.filter((r) => !r.passed); if (failures.length > 0) { - const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => a.id).join(",")}`).join("\n"); + const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => `${a.id}:${a.reason ?? ""}`).join(",")}`).join("\n"); throw new Error(`scenario failures:\n${detail}`); } - expect(summary.passed).toBe(24); + expect(summary.total).toBe(35); + expect(summary.passed).toBe(35); }, 120000); }); @@ -162,17 +205,20 @@ describe("CL-01 negative controls", () => { } }, 120000); - test("runNegativeControls summary counts rejections", async () => { + test("runNegativeControls summary names rejected controls explicitly", async () => { const summary = await runNegativeControls(); expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); - expect(summary.passed).toBe(summary.total); + expect(summary.rejected).toBe(summary.total); + expect(summary.passed).toBe(summary.rejected); }, 120000); }); describe("CL-01 scenario discovery API", () => { test("listScenarioIds returns stable mapping", () => { const ids = listScenarioIds(); - expect(ids.length).toBe(24); - expect(ids.sort()).toEqual([...ids].sort()); + const again = listScenarioIds(); + expect(ids.length).toBe(35); + expect(again).toEqual(ids); + expect(new Set(ids).size).toBe(ids.length); }); }); diff --git a/tests/openai-chat-tool-result-images.test.ts b/tests/openai-chat-tool-result-images.test.ts index b2c389e1fc..d94fa2bb5d 100644 --- a/tests/openai-chat-tool-result-images.test.ts +++ b/tests/openai-chat-tool-result-images.test.ts @@ -3,9 +3,9 @@ import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../src/types"; // Issue #888: role:"tool" content is text-only on chat-completions providers, so images inside a -// tool result were flattened to an "[image]" marker and vision-capable routed models hallucinated -// what they never saw. Tool-result images now ride in a follow-up user vision message released when -// the tool round closes, without splitting the round (strict providers reject interleaved users). +// tool result ride in a follow-up user vision message released when the tool round closes. When +// text is present, the tool row carries only that literal text; image markers are not duplicated +// into a row whose actual images are delivered separately. const provider: OcxProviderConfig = { adapter: "openai-chat", @@ -56,7 +56,6 @@ function toolResult(callId: string, name: string, content: string | OcxContentPa return { role: "toolResult", toolCallId: callId, toolName: name, content, isError: false, timestamp: 0 }; } -/** The carrier is a user message whose parts start with an "[ocx]" text label followed by image_url parts. */ function isImageCarrier(msg: ChatMsg): boolean { if (msg.role !== "user" || !Array.isArray(msg.content)) return false; const [head, ...rest] = msg.content; @@ -64,7 +63,6 @@ function isImageCarrier(msg: ChatMsg): boolean { && rest.length > 0 && rest.every(p => p.type === "image_url"); } -/** Every role:"tool" message must sit in an unbroken block right after its assistant tool_calls message. */ function assertRoundsUnbroken(messages: ChatMsg[]): void { for (let i = 0; i < messages.length; i++) { const m = messages[i]; @@ -85,12 +83,12 @@ test("tool-result images ride a follow-up user message; text, detail, and https { type: "text", text: "1 match found" }, { type: "image", imageUrl: IMAGE_URL, detail: "high" }, { type: "image", imageUrl: "https://example.test/shot.png" }, - { type: "image", imageUrl: "" }, // empty file_id shape: keeps its marker, never reaches the carrier + { type: "image", imageUrl: "" }, ]), ]); assertRoundsUnbroken(messages); const tool = messages.find(m => m.role === "tool")!; - expect(tool.content).toBe("1 match found[image][image][image]"); + expect(tool.content).toBe("1 match found"); const carrier = messages.find(isImageCarrier)!; expect(carrier).toBeDefined(); expect(messages.indexOf(carrier)).toBe(messages.indexOf(tool) + 1); @@ -109,7 +107,7 @@ test("images from a multi-call round flush once, only after the whole round clos ]); assertRoundsUnbroken(messages); const toolIdx = messages.map((m, i) => (m.role === "tool" ? i : -1)).filter(i => i >= 0); - expect(toolIdx).toEqual([toolIdx[0], toolIdx[0] + 1]); // nothing interleaves the round + expect(toolIdx).toEqual([toolIdx[0], toolIdx[0] + 1]); const carriers = messages.filter(isImageCarrier); expect(carriers.length).toBe(1); expect(messages.indexOf(carriers[0])).toBe(toolIdx[1] + 1); From 5639c6ce270af74d3c7ff39fc04a96409589a933 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:11:44 +0200 Subject: [PATCH 06/22] fix(lab): close remaining Protocol V1 gaps --- .../022_protocol_v1_cases.json | 2 +- src/lab/conformance/executor.ts | 48 +++++++++++++------ .../fixtures/protocol-v1-cases.json | 2 +- src/lab/conformance/sse-normalize.ts | 3 +- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json index 491aee9815..e0c55919cb 100644 --- a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json +++ b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json @@ -274,7 +274,7 @@ "assertions": [ { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, - { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/6/data/item/phase", "expected": "final_answer", "required": true } ] }, { diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 1e3d349d61..b18dab96d9 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -420,7 +420,13 @@ async function runReasoningReplay( observation: NormalizedObservation, vector: Record, ): Promise { - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + const provider: OcxProviderConfig = { + ...fixtureProviderConfig("openai-responses"), + // This Protocol V1 vector exercises a Responses-compatible target that accepts provider + // replay fields verbatim. The adapter must therefore preserve raw reasoning content. + preserveResponsesReasoningContent: true, + }; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); try { const turn1 = vector.turn1 as { reasoning: { id: string; text: string; signature: string }; @@ -429,20 +435,32 @@ async function runReasoningReplay( const turn2 = vector.turn2 as { toolResult: { callId: string; output: string } }; const first = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: "PING", stream: false }), { headers: new Headers() }); recordUpstreamRequest(observation, JSON.parse(first.body)); - const replayInput = [ - { - type: "reasoning", - id: turn1.reasoning.id, - content: [{ type: "reasoning_text", text: turn1.reasoning.text }], - signature: turn1.reasoning.signature, - }, - { - type: "function_call_output", - call_id: turn2.toolResult.callId, - output: turn2.toolResult.output, - }, - ]; - const second = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: replayInput, stream: false }), { headers: new Headers() }); + + const replayBody = { + model: "fixture-model", + input: [ + { + type: "reasoning", + id: turn1.reasoning.id, + content: [{ type: "reasoning_text", text: turn1.reasoning.text }], + signature: turn1.reasoning.signature, + }, + { + type: "function_call_output", + call_id: turn2.toolResult.callId, + output: turn2.toolResult.output, + }, + ], + stream: false, + }; + // Adapter vectors feed their documented boundary fields directly into the selected adapter. + // Keep a valid parsed shell for typed adapter metadata, while _rawBody carries the exact + // Responses replay shape whose text/signature preservation is under test. + const parsedReplay = { + ...parseRequest({ model: "fixture-model", input: "PING", stream: false }), + _rawBody: replayBody, + }; + const second = await adapter.buildRequest(parsedReplay, { headers: new Headers() }); recordUpstreamRequest(observation, JSON.parse(second.body)); return observation; } finally { diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json index 491aee9815..e0c55919cb 100644 --- a/src/lab/conformance/fixtures/protocol-v1-cases.json +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -274,7 +274,7 @@ "assertions": [ { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, - { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/6/data/item/phase", "expected": "final_answer", "required": true } ] }, { diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts index 03fd2eb0b8..0d3a9525e8 100644 --- a/src/lab/conformance/sse-normalize.ts +++ b/src/lab/conformance/sse-normalize.ts @@ -38,7 +38,8 @@ export function normalizeSseBytes(bytes: Uint8Array, sourceProtocol: string): No events.push({ event: eventName ?? "malformed", data: joined, ordinal: ordinal++ }); continue; } - if (parsed === null || typeof parsed !== "object") continue; + // Protocol V1 treats null, scalar, and array data values as padding. + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) continue; const inferred = eventName ?? (typeof (parsed as { type?: unknown }).type === "string" ? (parsed as { type: string }).type : "message"); From b16670fe4ada4f52200e215e3164482a793fb4a1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:13:42 +0200 Subject: [PATCH 07/22] fix(lab): harden tool-call projections --- src/lab/conformance/observation.ts | 60 ++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index 287467802f..0f1cca02d4 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -56,14 +56,21 @@ function parseToolArguments(raw: unknown, kind: "function" | "custom"): unknown return raw; } -/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ -export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { +interface ToolCallProjectionResult { + calls: ToolCallProjection[]; + sawCallItems: boolean; + duplicateIds: boolean; +} + +function projectToolCallsDetailed(output: unknown[]): ToolCallProjectionResult { const calls: ToolCallProjection[] = []; let ordinal = 0; + let sawCallItems = false; for (const item of output) { if (!item || typeof item !== "object") continue; const rec = item as Record; if (rec.type === "function_call") { + sawCallItems = true; const id = rec.call_id ?? rec.id; const name = rec.name; if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; @@ -71,6 +78,7 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio if (args === null) continue; calls.push({ id, name, arguments: args, kind: "function", ordinal: ordinal++ }); } else if (rec.type === "custom_tool_call") { + sawCallItems = true; const id = rec.call_id ?? rec.id; const name = rec.name; if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; @@ -84,10 +92,16 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio } } const ids = calls.map((call) => call.id); - return new Set(ids).size === ids.length ? calls : []; + const duplicateIds = new Set(ids).size !== ids.length; + return { calls: duplicateIds ? [] : calls, sawCallItems, duplicateIds }; } -export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { +/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ +export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { + return projectToolCallsDetailed(output).calls; +} + +function projectToolCallsFromEventsDetailed(events: NormalizedEvent[]): ToolCallProjectionResult { const output: unknown[] = []; for (const ev of events) { if (ev.event === "response.output_item.done" && ev.data && typeof ev.data === "object") { @@ -96,7 +110,11 @@ export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallP if (item && typeof item === "object") output.push(item); } } - return projectToolCallsFromOutput(output); + return projectToolCallsDetailed(output); +} + +export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { + return projectToolCallsFromEventsDetailed(events).calls; } export function projectMcpCalls(toolCalls: ToolCallProjection[]): Array<{ namespace: string; name: string }> { @@ -168,14 +186,14 @@ export function finalizeObservation( json: unknown = null, status = 200, ): void { - const eventToolCalls = projectToolCallsFromEvents(events); - const resolvedToolCalls = eventToolCalls.length > 0 - ? eventToolCalls - : projectToolCallsFromOutput( - json && typeof json === "object" && !Array.isArray(json) - ? ((json as { output?: unknown[] }).output ?? []) - : [], - ); + const eventProjection = projectToolCallsFromEventsDetailed(events); + const jsonProjection = projectToolCallsDetailed( + json && typeof json === "object" && !Array.isArray(json) + ? ((json as { output?: unknown[] }).output ?? []) + : [], + ); + const selectedProjection = eventProjection.sawCallItems ? eventProjection : jsonProjection; + const resolvedToolCalls = selectedProjection.calls; const terminal = deriveTerminal(events); setClientResponse(observation, { events, @@ -186,6 +204,7 @@ export function finalizeObservation( json, status, }); + observation.verifiers.duplicate_tool_call_ids = selectedProjection.duplicateIds; } export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { @@ -232,25 +251,26 @@ function evaluateCallResultOrder(observation: NormalizedObservation): string { const request = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; const input = request?.input; if (!Array.isArray(input)) return "fail"; - let pendingCallId: string | undefined; + const pendingCallIds = new Set(); + let callCount = 0; let resultCount = 0; for (const item of input) { if (!item || typeof item !== "object") continue; const record = item as { type?: string; call_id?: unknown }; if (record.type === "function_call") { - if (pendingCallId !== undefined) return "fail"; if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; - pendingCallId = record.call_id; + if (pendingCallIds.has(record.call_id)) return "fail"; + pendingCallIds.add(record.call_id); + callCount++; continue; } if (record.type === "function_call_output") { - if (pendingCallId === undefined) return "fail"; - if (typeof record.call_id !== "string" || record.call_id !== pendingCallId) return "fail"; + if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; + if (!pendingCallIds.delete(record.call_id)) return "fail"; resultCount++; - if (resultCount > 1) return "fail"; } } - return pendingCallId !== undefined && resultCount === 1 ? "pass" : "fail"; + return callCount > 0 && resultCount === callCount && pendingCallIds.size === 0 ? "pass" : "fail"; } function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { From f5ddee4f08cbe5b2f849467114ec0ca5f832b35d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:01 +0200 Subject: [PATCH 08/22] fix(lab): fail closed on malformed controls --- src/lab/conformance/executor.ts | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index b18dab96d9..ec3f32a479 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -529,9 +529,29 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; + finalizeObservation(observation, events, json, responseStatus); + attachVerifiers(observation, caseRecord); + return observation; + } finally { + adapter.dispose(); + } + } + let events: ReturnType; - let json: Record | null = null; - let responseStatus = 200; if (upstreamProtocol === "openai-chat") { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); @@ -600,21 +620,7 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; - } finally { - adapter.dispose(); - } - } - - finalizeObservation(observation, events, json, responseStatus); + finalizeObservation(observation, events, null, 200); attachVerifiers(observation, caseRecord); return observation; } @@ -651,6 +657,9 @@ export async function runScenario(caseRecord: CaseRecord): Promise assertionResults.find((r) => r.id === id)?.passed === true); const expectedFailureMatched = controlPassed && requiredFailures.length === 0; return { From 78144847a0c89a3186b9d67efbd71fb5fbb804d1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:22 +0200 Subject: [PATCH 09/22] test(lab): cover review regression edges --- tests/cl01-review-regressions.test.ts | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/cl01-review-regressions.test.ts diff --git a/tests/cl01-review-regressions.test.ts b/tests/cl01-review-regressions.test.ts new file mode 100644 index 0000000000..1ae401c3de --- /dev/null +++ b/tests/cl01-review-regressions.test.ts @@ -0,0 +1,74 @@ +import { expect, test } from "bun:test"; +import { runScenario } from "../src/lab/conformance/executor"; +import { loadCaseAuthority } from "../src/lab/conformance/manifest"; +import { emptyObservation, finalizeObservation } from "../src/lab/conformance/observation"; +import type { CaseRecord, NormalizedEvent } from "../src/lab/conformance/types"; + +test("duplicate event tool-call ids stay rejected instead of falling back to JSON", () => { + const observation = emptyObservation(); + const events: NormalizedEvent[] = [ + { + event: "response.output_item.done", + ordinal: 0, + data: { item: { type: "function_call", call_id: "dup", name: "a", arguments: "{}" } }, + }, + { + event: "response.output_item.done", + ordinal: 1, + data: { item: { type: "function_call", call_id: "dup", name: "b", arguments: "{}" } }, + }, + ]; + finalizeObservation(observation, events, { + output: [{ type: "function_call", call_id: "fallback", name: "c", arguments: "{}" }], + }); + + expect(observation.client.response.toolCalls).toEqual([]); + expect(observation.verifiers.duplicate_tool_call_ids).toBe(true); +}); + +test("call/result-order verifier accepts multiple correlated pairs", async () => { + const authority = loadCaseAuthority(); + const base = authority.cases.find((c) => c.id === "codex-core.protocol.tool-continuation")!; + const fixture = { + turn1: { + output: [ + { type: "function_call", id: "fc_a", call_id: "call_a", name: "a", arguments: "{}" }, + { type: "function_call", id: "fc_b", call_id: "call_b", name: "b", arguments: "{}" }, + ], + }, + turn2: { + input: [ + { type: "function_call_output", call_id: "call_a", output: "A" }, + { type: "function_call_output", call_id: "call_b", output: "B" }, + ], + }, + }; + const scenario: CaseRecord = { + ...structuredClone(base), + fixture: { ...base.fixture, bytesUtf8: JSON.stringify(fixture) }, + assertions: [{ + id: "order", + operator: "json_path_equals", + selector: "/verifiers/call_result_order", + expected: "pass", + required: true, + }], + }; + + const result = await runScenario(scenario); + expect(result.passed).toBe(true); +}); + +test("expected-failure controls with no assertion ids fail as malformed manifests", async () => { + const authority = loadCaseAuthority(); + const base = authority.cases.find((c) => c.id === "vision-core.protocol.modality-gate")!; + const scenario: CaseRecord = { + ...structuredClone(base), + expectedFailure: { ...base.expectedFailure!, assertionIds: [] }, + }; + + const result = await runScenario(scenario); + expect(result.passed).toBe(false); + expect(result.classification).toBe("harness_failure"); + expect(result.diagnostics.join(" ")).toContain("lists no assertionIds"); +}); From 5aab6d9dfa87f6f7230dcf3b8877c2d2555d6ec0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:47 +0200 Subject: [PATCH 10/22] test(openai-chat): pin native CL-01 regressions --- ...l01-openai-chat-review-regressions.test.ts | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/cl01-openai-chat-review-regressions.test.ts diff --git a/tests/cl01-openai-chat-review-regressions.test.ts b/tests/cl01-openai-chat-review-regressions.test.ts new file mode 100644 index 0000000000..0d755e0dd9 --- /dev/null +++ b/tests/cl01-openai-chat-review-regressions.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; +import type { OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../src/types"; + +const baseProvider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://api.openai.com/v1", + apiKey: "sk-test", + authMode: "key", +}; + +function bodyFor(provider: OcxProviderConfig, parsed: OcxParsedRequest): Record { + const request = createOpenAIChatAdapter(provider).buildRequest(parsed) as { body: string }; + return JSON.parse(request.body) as Record; +} + +function assistantToolCall(id: string, name: string): OcxMessage { + return { + role: "assistant", + content: [{ type: "toolCall", id, name, arguments: {} }], + timestamp: 0, + }; +} + +test("native OpenAI defers developer guidance until pending tool results are complete", () => { + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { + messages: [ + { role: "user", content: "go", timestamp: 0 }, + assistantToolCall("call_1", "lookup"), + { role: "developer", content: "after the tool", timestamp: 0 }, + { role: "toolResult", toolCallId: "call_1", toolName: "lookup", content: "ok", isError: false, timestamp: 0 }, + ], + }, + stream: false, + options: {}, + }; + + const messages = bodyFor(baseProvider, parsed).messages as Array>; + const assistantIndex = messages.findIndex((m) => m.role === "assistant"); + expect(messages[assistantIndex + 1]).toMatchObject({ role: "tool", tool_call_id: "call_1" }); + expect(messages[assistantIndex + 2]).toEqual({ role: "developer", content: "after the tool" }); +}); + +test("native OpenAI Chat uses reasoning_effort instead of the gateway reasoning object", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { high: "high", none: "none" }, + }; + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { messages: [{ role: "user", content: "think", timestamp: 0 }] }, + stream: false, + options: { reasoning: "high" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning_effort).toBe("high"); + expect(body.reasoning).toBeUndefined(); +}); + +test("native OpenAI Chat represents disabled reasoning with reasoning_effort none", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { none: "none" }, + }; + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { messages: [{ role: "user", content: "short", timestamp: 0 }] }, + stream: false, + options: { reasoning: "none" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning_effort).toBe("none"); + expect(body.reasoning).toBeUndefined(); +}); + +test("non-native gateway targets keep their reasoning object", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + baseUrl: "https://gateway.example.test/v1", + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { high: "adaptive" }, + }; + const parsed: OcxParsedRequest = { + modelId: "fixture-model", + context: { messages: [{ role: "user", content: "think", timestamp: 0 }] }, + stream: false, + options: { reasoning: "high" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning).toEqual({ enabled: true, effort: "adaptive" }); + expect(body.reasoning_effort).toBeUndefined(); +}); + +test("paired image-only tool results retain their flattened tool-row fallback", () => { + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { + messages: [ + assistantToolCall("call_img", "shot"), + { + role: "toolResult", + toolCallId: "call_img", + toolName: "shot", + content: [{ type: "image", imageUrl: "data:image/png;base64,aGVsbG8=" }], + isError: false, + timestamp: 0, + }, + ], + }, + stream: false, + options: {}, + }; + + const messages = bodyFor({ ...baseProvider, baseUrl: "https://example.test/v1" }, parsed).messages as Array>; + expect(messages.find((m) => m.role === "tool")?.content).toBe("[image]"); +}); From f79fc9eac31915376ce5c703d29b937337f1ea50 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:17:44 +0200 Subject: [PATCH 11/22] fix(openai-chat): use native reasoning effort field --- src/adapters/openai-chat.ts | 45 +++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 31a7d1726e..6554ee067b 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -693,24 +693,41 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (parsed.options.stopSequences !== undefined) body.stop = parsed.options.stopSequences; const reasoningDisabled = modelInList(provider.noReasoningModels, parsed.modelId); const reasoningEffort = mapReasoningEffort(provider, parsed.modelId, parsed.options.reasoning); + const nativeOpenAI = isNativeOpenAIChatTarget(provider); let reasoningLog: AdapterRequest["reasoningLog"]; if (!reasoningDisabled && provider.reasoningWireFormat === "gateway-object" && parsed.options.reasoning === "none") { - body.reasoning = { enabled: false }; - reasoningLog = { - effectiveEffort: "none", - wireField: "reasoning.enabled", - wireValue: false, - }; - } else if (reasoningEffort !== undefined) { - if (provider.reasoningWireFormat === "gateway-object") { - body.reasoning = isNativeOpenAIChatTarget(provider) - ? { effort: reasoningEffort } - : { enabled: true, effort: reasoningEffort }; + if (nativeOpenAI) { + body.reasoning_effort = "none"; reasoningLog = { - effectiveEffort: reasoningEffort, - wireField: "reasoning.effort", - wireValue: reasoningEffort, + effectiveEffort: "none", + wireField: "reasoning_effort", + wireValue: "none", }; + } else { + body.reasoning = { enabled: false }; + reasoningLog = { + effectiveEffort: "none", + wireField: "reasoning.enabled", + wireValue: false, + }; + } + } else if (reasoningEffort !== undefined) { + if (provider.reasoningWireFormat === "gateway-object") { + if (nativeOpenAI) { + body.reasoning_effort = reasoningEffort; + reasoningLog = { + effectiveEffort: reasoningEffort, + wireField: "reasoning_effort", + wireValue: reasoningEffort, + }; + } else { + body.reasoning = { enabled: true, effort: reasoningEffort }; + reasoningLog = { + effectiveEffort: reasoningEffort, + wireField: "reasoning.effort", + wireValue: reasoningEffort, + }; + } } else if (modelInList(provider.thinkingBudgetModels, parsed.modelId)) { const budget = thinkingBudgetForEffort(parsed, reasoningEffort, maxTokens); if (budget !== undefined) { From 554f6f6d8b4b2007f5c7238b5176275a24fe041e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:18:12 +0200 Subject: [PATCH 12/22] test(claude): pin initial failure framing --- ...claude-outbound-review-regressions.test.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/cl01-claude-outbound-review-regressions.test.ts diff --git a/tests/cl01-claude-outbound-review-regressions.test.ts b/tests/cl01-claude-outbound-review-regressions.test.ts new file mode 100644 index 0000000000..72849097d8 --- /dev/null +++ b/tests/cl01-claude-outbound-review-regressions.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { responsesSseToAnthropicSse } from "../src/claude/outbound"; +import { createTranslatorBudget } from "../src/lib/translator-budget"; + +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text + decoder.decode(); + } finally { + reader.releaseLock(); + } +} + +test("an initial Responses failure becomes one Anthropic error event without a synthetic message start", async () => { + const frame = [ + "event: response.failed", + 'data: {"type":"response.failed","response":{"status":"failed","error":{"type":"server_error","code":"overloaded","message":"fixture"}}}', + "", + "", + ].join("\n"); + const upstream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(frame)); + controller.close(); + }, + }); + const budget = createTranslatorBudget(); + try { + const anthropic = responsesSseToAnthropicSse(upstream, "fixture-model", { + pingIntervalMs: 0, + translatorBudget: budget, + }); + const text = await collect(anthropic); + const eventNames = text + .split("\n") + .filter((line) => line.startsWith("event: ")) + .map((line) => line.slice("event: ".length)); + expect(eventNames).toEqual(["error"]); + } finally { + budget.dispose(); + } +}); From b86b303fe6ec40cabe9ca5c2ca34c24eb1ef0fd1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:23:37 +0200 Subject: [PATCH 13/22] fix(claude): preserve initial error framing --- src/claude/outbound.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/claude/outbound.ts b/src/claude/outbound.ts index 494720c497..275d605ee4 100644 --- a/src/claude/outbound.ts +++ b/src/claude/outbound.ts @@ -328,9 +328,14 @@ export function responsesSseToAnthropicSse( )))); return; } - ensureStarted(); - closeOpenBlock(); const type = upstreamDerived && isTransientUpstreamStatus(status) ? "overloaded_error" : undefined; + if (!started) { + // An initial upstream failure is an Anthropic error stream, not a partial message. + // Do not manufacture message_start/ping before the terminal error. + emit("error", anthropicErrorBody(status, message, type, code)); + return; + } + closeOpenBlock(); emit("error", anthropicErrorBody(status, message, type, code)); }; From ee5efaaf72d0ee9f14f5af9c47755488e24d598d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:35:12 +0200 Subject: [PATCH 14/22] fix(lab): preserve nonstream fixture fallback --- src/lab/conformance/executor.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index ec3f32a479..e00ecf6bb5 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -31,6 +31,16 @@ async function collectAdapterEvents(gen: AsyncGenerator): Promise< return events; } +export function nonstreamObservationJson( + parsedEvents: AdapterEvent[], + responseJson: Record, + model = "fixture-model", +): Record { + return parsedEvents.length > 0 + ? buildResponseJSON(parsedEvents, model) as Record + : responseJson; +} + async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model"): Promise<{ events: ReturnType; }> { @@ -542,7 +552,7 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; + const json = nonstreamObservationJson(parsedEvents, responseJson); finalizeObservation(observation, events, json, responseStatus); attachVerifiers(observation, caseRecord); return observation; From 28685431b8e195a4105da5b08d7d1493fe2743e9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:35:27 +0200 Subject: [PATCH 15/22] test(lab): preserve empty nonstream fixture fallback --- tests/cl01-review-regressions.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/cl01-review-regressions.test.ts b/tests/cl01-review-regressions.test.ts index 1ae401c3de..8cb219e8f8 100644 --- a/tests/cl01-review-regressions.test.ts +++ b/tests/cl01-review-regressions.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { runScenario } from "../src/lab/conformance/executor"; +import { nonstreamObservationJson, runScenario } from "../src/lab/conformance/executor"; import { loadCaseAuthority } from "../src/lab/conformance/manifest"; import { emptyObservation, finalizeObservation } from "../src/lab/conformance/observation"; import type { CaseRecord, NormalizedEvent } from "../src/lab/conformance/types"; @@ -72,3 +72,11 @@ test("expected-failure controls with no assertion ids fail as malformed manifest expect(result.classification).toBe("harness_failure"); expect(result.diagnostics.join(" ")).toContain("lists no assertionIds"); }); + +test("nonstream observation falls back to fixture JSON when parser emits no events", () => { + const fixture = { + id: "chatcmpl_fixture", + choices: [{ message: { role: "assistant", content: "OK" }, finish_reason: "stop" }], + }; + expect(nonstreamObservationJson([], fixture)).toBe(fixture); +}); From 3328f1b53226a055d7384bb084428b241cea1f2a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:40:11 +0200 Subject: [PATCH 16/22] fix(lab): isolate gateway reasoning fixture --- src/lab/conformance/executor.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index e00ecf6bb5..0d84f3094f 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -414,6 +414,9 @@ async function runReasoningEffortMapping( ): Promise { const provider: OcxProviderConfig = { ...fixtureProviderConfig("openai-chat"), + // This vector explicitly exercises the non-native gateway-object wire contract. + // Keep it off api.openai.com so native Chat's reasoning_effort branch is tested separately. + baseUrl: "http://127.0.0.1:1/v1", reasoningEffortMap: vector.reasoningEffortMap as Record, reasoningWireFormat: vector.reasoningWireFormat as OcxProviderConfig["reasoningWireFormat"], }; From b44a07a4f933a1a5c14ac8c2a99d884942b2d366 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:47:25 +0200 Subject: [PATCH 17/22] fix(lab): correct gateway reasoning authority --- .../_plan/260807_compatibility_lab/022_protocol_v1_cases.json | 2 +- src/lab/conformance/fixtures/protocol-v1-cases.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json index e0c55919cb..729ed6911b 100644 --- a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json +++ b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json @@ -374,7 +374,7 @@ "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "reasoning-effort", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"requested\":\"high\",\"reasoningEffortMap\":{\"high\":\"adaptive\"},\"reasoningWireFormat\":\"gateway-object\"}", "digest": "d9d5cce104809764d5edbc833088a0a9bb3b4d678a4f135353cc5fecf62e8b57" }, "assertions": [ - { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"effort":"adaptive"}, "required": true }, + { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"enabled":true,"effort":"adaptive"}, "required": true }, { "id": "legacy-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/reasoning_effort", "expected": true, "required": true } ] }, diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json index e0c55919cb..729ed6911b 100644 --- a/src/lab/conformance/fixtures/protocol-v1-cases.json +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -374,7 +374,7 @@ "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "reasoning-effort", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"requested\":\"high\",\"reasoningEffortMap\":{\"high\":\"adaptive\"},\"reasoningWireFormat\":\"gateway-object\"}", "digest": "d9d5cce104809764d5edbc833088a0a9bb3b4d678a4f135353cc5fecf62e8b57" }, "assertions": [ - { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"effort":"adaptive"}, "required": true }, + { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"enabled":true,"effort":"adaptive"}, "required": true }, { "id": "legacy-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/reasoning_effort", "expected": true, "required": true } ] }, From 0b851507bc644a596fb8539957320d72641fd970 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:52:52 +0200 Subject: [PATCH 18/22] test(claude): cover created-then-failed framing --- ...claude-outbound-review-regressions.test.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/cl01-claude-outbound-review-regressions.test.ts b/tests/cl01-claude-outbound-review-regressions.test.ts index 72849097d8..731f471e56 100644 --- a/tests/cl01-claude-outbound-review-regressions.test.ts +++ b/tests/cl01-claude-outbound-review-regressions.test.ts @@ -18,16 +18,24 @@ async function collect(stream: ReadableStream): Promise { } } -test("an initial Responses failure becomes one Anthropic error event without a synthetic message start", async () => { - const frame = [ - "event: response.failed", - 'data: {"type":"response.failed","response":{"status":"failed","error":{"type":"server_error","code":"overloaded","message":"fixture"}}}', - "", - "", - ].join("\n"); +test("a created-then-failed Responses stream becomes one Anthropic error event", async () => { + const frames = [ + [ + "event: response.created", + 'data: {"type":"response.created","response":{"id":"resp_fixture","status":"in_progress"}}', + "", + "", + ].join("\n"), + [ + "event: response.failed", + 'data: {"type":"response.failed","response":{"status":"failed","error":{"type":"server_error","code":"overloaded","message":"fixture"}}}', + "", + "", + ].join("\n"), + ].join(""); const upstream = new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode(frame)); + controller.enqueue(new TextEncoder().encode(frames)); controller.close(); }, }); From a82977d352cccf3f89d8a470a69cb6a9bf386f65 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:55:45 +0200 Subject: [PATCH 19/22] fix(claude): defer message start until semantic output --- src/claude/outbound.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/claude/outbound.ts b/src/claude/outbound.ts index 275d605ee4..c8b4d20df8 100644 --- a/src/claude/outbound.ts +++ b/src/claude/outbound.ts @@ -245,14 +245,13 @@ export function responsesSseToAnthropicSse( emit("message_start", { type: "message_start", message: messageSnapshot(model) }); emit("ping", { type: "ping" }); }; - // Idle keepalive (devlog 100): real Anthropic streams may interleave pings anywhere; - // synthesizing one during upstream silence protects remote deployments behind - // LB/NAT idle timeouts and covers slow first tokens. Cheap and spec-legal. + // Once a semantic Anthropic message has started, keepalive pings protect remote + // deployments behind LB/NAT idle timeouts. Transport-only Responses prelude frames + // must not manufacture a message before a possible initial error. if (pingIntervalMs > 0) { pingTimer = setInterval(() => { - if (terminated) return; + if (terminated || !started) return; try { - ensureStarted(); emit("ping", { type: "ping" }); } catch { /* controller torn down; the read loop is ending anyway */ } }, pingIntervalMs); @@ -342,11 +341,10 @@ export function responsesSseToAnthropicSse( const handleFrame = (eventName: string, data: Rec) => { switch (eventName) { case "response.created": - ensureStarted(); + // Transport prelude only. Start Anthropic framing on semantic output or completion. break; case "response.heartbeat": - ensureStarted(); - emit("ping", { type: "ping" }); + if (started) emit("ping", { type: "ping" }); break; case "response.output_text.delta": { if (typeof data.delta !== "string" || data.delta.length === 0) break; From 9288916c7cd47795e4f93af9931d8ac3ecf514ec Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:32:35 +0200 Subject: [PATCH 20/22] test(openai-chat): retain fallback for untransportable tool images --- tests/openai-chat-tool-result-images.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/openai-chat-tool-result-images.test.ts b/tests/openai-chat-tool-result-images.test.ts index d94fa2bb5d..307ba9b07f 100644 --- a/tests/openai-chat-tool-result-images.test.ts +++ b/tests/openai-chat-tool-result-images.test.ts @@ -4,8 +4,8 @@ import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig } // Issue #888: role:"tool" content is text-only on chat-completions providers, so images inside a // tool result ride in a follow-up user vision message released when the tool round closes. When -// text is present, the tool row carries only that literal text; image markers are not duplicated -// into a row whose actual images are delivered separately. +// text is present, the tool row carries that literal text plus fallback markers only for images +// that cannot be transported in the follow-up carrier. const provider: OcxProviderConfig = { adapter: "openai-chat", @@ -88,7 +88,7 @@ test("tool-result images ride a follow-up user message; text, detail, and https ]); assertRoundsUnbroken(messages); const tool = messages.find(m => m.role === "tool")!; - expect(tool.content).toBe("1 match found"); + expect(tool.content).toBe("1 match found[image]"); const carrier = messages.find(isImageCarrier)!; expect(carrier).toBeDefined(); expect(messages.indexOf(carrier)).toBe(messages.indexOf(tool) + 1); From 577819ba36415722a76d6cc8d654b65cf00756d4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:41:56 +0200 Subject: [PATCH 21/22] fix(openai-chat): preserve fallback for untransportable tool images --- src/adapters/openai-chat.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 6554ee067b..3e9b38d6c4 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -187,7 +187,10 @@ function isNativeOpenAIChatTarget(provider: OcxProviderConfig): boolean { function toolResultTextForWire(content: string | OcxContentPart[]): string { if (typeof content === "string") return content; const text = content.filter((p) => p.type === "text").map((p) => (p as OcxTextContent).text).join(""); - if (text) return text; + if (text) { + const untransportableImages = content.filter((p) => p.type === "image" && !p.imageUrl).length; + return `${text}${"[image]".repeat(untransportableImages)}`; + } return contentPartsToText(content); } From af962ad13a17dbbd4abc43a898cb8d0f1c6be770 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 05:45:48 +0200 Subject: [PATCH 22/22] test(claude): keep idle pings scoped to semantic streams --- tests/claude-outbound.test.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/claude-outbound.test.ts b/tests/claude-outbound.test.ts index 66250f683d..a804a1eefe 100644 --- a/tests/claude-outbound.test.ts +++ b/tests/claude-outbound.test.ts @@ -541,23 +541,17 @@ describe("claude outbound SSE", () => { expect(events.at(-1)).toMatchObject({ name: "error", data: { error: { type: "overloaded_error" } } }); }); - test("idle keepalive pings flow during upstream silence", async () => { - // Upstream: created frame, a stretch of silence, then a clean completion. - // - // The silence is deliberately many intervals long. At 90ms with a 25ms ping the - // margin was 3.6 intervals against a >=3 assertion, so a single coalesced timer on - // a loaded runner failed it — which is how this went red on macos-latest while - // passing everywhere else. Timer scheduling is best-effort, not exact. - // - // The assertion below is unchanged. What changed is the headroom: the test still - // proves idle pings flow during silence, it just no longer depends on the runner - // delivering timers on schedule. + test("idle keepalive pings flow after semantic output during upstream silence", async () => { + // response.created is transport-only and must not start Anthropic framing because the + // next semantic frame could still be an initial error. Once real output starts the + // Anthropic message, periodic pings keep an otherwise idle connection alive. const PING_INTERVAL_MS = 25; const SILENCE_MS = 300; const encoder = new TextEncoder(); const upstream = new ReadableStream({ async start(controller) { controller.enqueue(encoder.encode(sse("response.created", { response: {} }))); + controller.enqueue(encoder.encode(sse("response.output_text.delta", { delta: "x" }))); await new Promise(r => setTimeout(r, SILENCE_MS)); controller.enqueue(encoder.encode(sse("response.completed", { response: { status: "completed", usage: { input_tokens: 1, output_tokens: 1 } } }))); controller.close(); @@ -565,7 +559,7 @@ describe("claude outbound SSE", () => { }); const events = await collectEvents(responsesSseToAnthropicSse(upstream, "m", { pingIntervalMs: PING_INTERVAL_MS })); const pings = events.filter(e => e.name === "ping").length; - expect(pings).toBeGreaterThanOrEqual(3); // startup ping + >=2 idle pings + expect(pings).toBeGreaterThanOrEqual(3); // startup ping + >=2 idle pings after semantic start expect(events.at(-1)!.name).toBe("message_stop"); });