From 881e54133c3c66d93a5cb8d3012da7a49dc9c6d7 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 28 Jul 2026 02:19:12 +0200 Subject: [PATCH 1/2] Support raw view output in replay fixtures Canonicalize view tool snapshots to raw file content and add an exact-first replay fallback for CLI versions that still prefix sequential line numbers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3c5c3fb3-1b45-495a-84d6-90521feaff29 --- test/harness/replayingCapiProxy.test.ts | 158 +++++++++++ test/harness/replayingCapiProxy.ts | 246 +++++++++++++----- .../should_create_a_new_file.yaml | 2 +- .../should_edit_a_file_successfully.yaml | 5 +- .../should_read_file_with_line_range.yaml | 6 +- ...ient_cwd_for_default_workingdirectory.yaml | 2 +- ...ect_order_for_tool_using_conversation.yaml | 2 +- ..._execution_events_with_correct_fields.yaml | 2 +- ...e_order_in_getmessages_after_tool_use.yaml | 2 +- ...nvoke_both_hooks_for_single_tool_call.yaml | 2 +- ...tool_use_hook_after_model_runs_a_tool.yaml | 2 +- ..._tool_use_hook_when_model_runs_a_tool.yaml | 2 +- ...ttooluse_hooks_for_a_single_tool_call.yaml | 2 +- ...osttooluse_hooks_for_single_tool_call.yaml | 2 +- ...ttooluse_hook_after_model_runs_a_tool.yaml | 2 +- ...retooluse_hook_when_model_runs_a_tool.yaml | 2 +- ...le_creation_then_reading_across_turns.yaml | 2 +- ..._use_tool_results_from_previous_turns.yaml | 4 +- ...rmission_handler_for_write_operations.yaml | 4 +- ...rmission_handler_for_write_operations.yaml | 4 +- .../should_send_with_file_attachment.yaml | 2 +- .../should_accept_message_attachments.yaml | 2 +- ...ly_workingdirectory_on_session_resume.yaml | 2 +- ...e_workingdirectory_for_tool_execution.yaml | 2 +- ...ooluse_hooks_for_sub_agent_tool_calls.yaml | 2 +- ...form_modifications_to_section_content.yaml | 2 +- ...nsform_callbacks_with_section_content.yaml | 2 +- ...tic_overrides_and_transforms_together.yaml | 2 +- .../tools/invokes_built_in_tools.yaml | 2 +- 29 files changed, 368 insertions(+), 103 deletions(-) diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index c5747a3067..a4b685ca7d 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -724,6 +724,107 @@ Always include PINEAPPLE_COCONUT_42. }); } + async function replayToolResult( + toolName: string, + requestContent: string, + savedResults: Array<{ content: string; response: string }>, + savedErrors: Array<{ + content: string; + status: number; + message: string; + }> = [], + ): Promise { + const toolArguments = toolName === "view" ? '{"path":"file.txt"}' : "{}"; + const createMessages = ( + content: string, + response?: string, + ): NormalizedData["conversations"][number]["messages"] => { + const messages: NormalizedData["conversations"][number]["messages"] = [ + { role: "system", content: "${system}" }, + { role: "user", content: "Use the tool" }, + { + role: "assistant", + tool_calls: [ + { + id: "toolcall_0", + type: "function", + function: { + name: toolName, + arguments: toolArguments, + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content, + }, + ]; + if (response !== undefined) { + messages.push({ role: "assistant", content: response }); + } + return messages; + }; + const cachePath = path.join(tempDir, "cache.yaml"); + const cacheContent = yaml.stringify({ + models: ["test-model"], + errors: savedErrors.map((savedError) => ({ + model: "test-model", + status: savedError.status, + message: savedError.message, + messages: createMessages(savedError.content), + })), + conversations: savedResults.map((savedResult) => ({ + messages: createMessages(savedResult.content, savedResult.response), + })), + } satisfies NormalizedData); + await writeFile(cachePath, cacheContent); + + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + + try { + const response = await makeRequest(proxyUrl, "/chat/completions", { + body: { + model: "test-model", + messages: [ + { role: "system", content: "System prompt" }, + { role: "user", content: "Use the tool" }, + { + role: "assistant", + tool_calls: [ + { + id: "request-tool-call", + type: "function", + function: { + name: toolName, + arguments: toolArguments, + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "request-tool-call", + content: requestContent, + }, + ], + }, + }); + + expect(response.status).toBe(200); + return (JSON.parse(response.body) as ChatCompletion).choices[0].message + .content; + } finally { + await proxy.stop(); + } + } + test("returns cached response when request matches prefix", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ @@ -820,6 +921,63 @@ Always include PINEAPPLE_COCONUT_42. } }); + test("matches legacy numbered view results against raw cached content", async () => { + const result = await replayToolResult( + "view", + "42. alpha\r\n43. beta\r\n44.", + [{ content: "alpha\r\nbeta", response: "legacy view matched" }], + ); + + expect(result).toBe("legacy view matched"); + }); + + test("prefers an exact naturally numbered view result match", async () => { + const result = await replayToolResult("view", "1. alpha\n2. beta", [ + { content: "alpha\nbeta", response: "legacy fallback" }, + { content: "1. alpha\n2. beta", response: "exact match" }, + ]); + + expect(result).toBe("exact match"); + }); + + test("prefers an exact response over a legacy-fallback error", async () => { + const result = await replayToolResult( + "view", + "1. alpha\n2. beta", + [{ content: "1. alpha\n2. beta", response: "exact response" }], + [ + { + content: "alpha\nbeta", + status: 429, + message: "legacy fallback error", + }, + ], + ); + + expect(result).toBe("exact response"); + }); + + test("does not use legacy view compatibility for other tools", async () => { + const result = await replayToolResult("grep", "1. alpha\n2. beta", [ + { content: "alpha\nbeta", response: "incorrect fallback" }, + { content: "1. alpha\n2. beta", response: "exact grep match" }, + ]); + + expect(result).toBe("exact grep match"); + }); + + test("does not strip non-sequential view results", async () => { + const result = await replayToolResult("view", "1. alpha\n3. gamma", [ + { content: "alpha\ngamma", response: "incorrect fallback" }, + { + content: "1. alpha\n3. gamma", + response: "exact non-sequential match", + }, + ]); + + expect(result).toBe("exact non-sequential match"); + }); + test("matches shell tool results with shell ID completion markers", async () => { const originalShellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 5e07449f73..1422a6d601 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -454,75 +454,80 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { const streamingIsRequested = (JSON.parse(normalizedBody) as { stream?: boolean }).stream === true; - const savedError = await findSavedChatCompletionError( - state.storedData, - normalizedBody, - state.workDir, - state.toolResultNormalizers, - ); + for (const allowLegacyViewFallback of [false, true]) { + const savedError = await findSavedChatCompletionError( + state.storedData, + normalizedBody, + state.workDir, + state.toolResultNormalizers, + allowLegacyViewFallback, + ); - if (savedError) { - const headers = { - "content-type": "application/json", - ...commonResponseHeaders, - ...(savedError.retryAfterSeconds !== undefined - ? { "retry-after": String(savedError.retryAfterSeconds) } - : {}), - }; - options.onResponseStart(savedError.status, headers); - options.onData( - Buffer.from( - JSON.stringify( - (protocol.errorBody ?? openAIErrorBody)( - savedError.code, - savedError.message ?? "Rate limited by test snapshot", + if (savedError) { + const headers = { + "content-type": "application/json", + ...commonResponseHeaders, + ...(savedError.retryAfterSeconds !== undefined + ? { "retry-after": String(savedError.retryAfterSeconds) } + : {}), + }; + options.onResponseStart(savedError.status, headers); + options.onData( + Buffer.from( + JSON.stringify( + (protocol.errorBody ?? openAIErrorBody)( + savedError.code, + savedError.message ?? "Rate limited by test snapshot", + ), ), ), - ), - ); - options.onResponseEnd(); - return; - } - - const savedResponse = await findSavedChatCompletionResponse( - state.storedData, - normalizedBody, - state.workDir, - state.toolResultNormalizers, - ); - - if (savedResponse) { - await this.respondWithProtocol( - options, - protocol, - savedResponse, - streamingIsRequested, - commonResponseHeaders, - ); - - return; - } + ); + options.onResponseEnd(); + return; + } - // Check if this request matches a snapshot with no response (e.g., timeout tests). - // If so, hang forever so the client-side timeout can trigger. - if ( - await isRequestOnlySnapshot( + const savedResponse = await findSavedChatCompletionResponse( state.storedData, normalizedBody, state.workDir, state.toolResultNormalizers, - ) - ) { - const headers = { - "content-type": streamingIsRequested - ? "text/event-stream" - : "application/json", - ...commonResponseHeaders, - }; - options.onResponseStart(200, headers); - // Never call onResponseEnd - hang indefinitely for timeout tests. - // Returning here keeps the HTTP response open without leaking a pending Promise. - return; + allowLegacyViewFallback, + ); + + if (savedResponse) { + await this.respondWithProtocol( + options, + protocol, + savedResponse, + streamingIsRequested, + commonResponseHeaders, + ); + + return; + } + + // Check if this request matches a snapshot with no response (e.g., timeout tests). + // If so, hang forever so the client-side timeout can trigger. + if ( + await isRequestOnlySnapshot( + state.storedData, + normalizedBody, + state.workDir, + state.toolResultNormalizers, + allowLegacyViewFallback, + ) + ) { + const headers = { + "content-type": streamingIsRequested + ? "text/event-stream" + : "application/json", + ...commonResponseHeaders, + }; + options.onResponseStart(200, headers); + // Never call onResponseEnd - hang indefinitely for timeout tests. + // Returning here keeps the HTTP response open without leaking a pending Promise. + return; + } } } @@ -672,7 +677,7 @@ function diagnoseMatchFailure( // Find the first message that doesn't match let mismatchIndex = -1; for (let i = 0; i < requestMessages.length; i++) { - if (JSON.stringify(requestMessages[i]) !== JSON.stringify(saved[i])) { + if (!messagesEqualForReplay(requestMessages, saved, i, true)) { mismatchIndex = i; break; } @@ -762,6 +767,7 @@ async function findSavedChatCompletionResponse( requestBody: string | undefined, workDir: string, toolResultNormalizers: ToolResultNormalizer[], + allowLegacyViewFallback: boolean, ): Promise { // Normalize the incoming request the same way we normalize for caching const normalized = await parseAndNormalizeRequest( @@ -775,11 +781,11 @@ async function findSavedChatCompletionResponse( throw new Error("Unable to determine model from request"); } - // Now find a matching cached conversation (i.e., one for which this request is a prefix) for (const conversation of storedData.conversations) { const replyIndex = findAssistantIndexAfterPrefix( requestMessages, conversation.messages, + allowLegacyViewFallback, ); if (replyIndex !== undefined) { return createOpenAIResponse( @@ -799,6 +805,7 @@ async function findSavedChatCompletionError( requestBody: string | undefined, workDir: string, toolResultNormalizers: ToolResultNormalizer[], + allowLegacyViewFallback: boolean, ): Promise { const normalized = await parseAndNormalizeRequest( requestBody, @@ -814,8 +821,13 @@ async function findSavedChatCompletionError( } if ( requestMessages.length === error.messages.length && - requestMessages.every( - (msg, i) => JSON.stringify(msg) === JSON.stringify(error.messages[i]), + requestMessages.every((_msg, i) => + messagesEqualForReplay( + requestMessages, + error.messages, + i, + allowLegacyViewFallback, + ), ) ) { return error; @@ -832,6 +844,7 @@ async function isRequestOnlySnapshot( requestBody: string | undefined, workDir: string, toolResultNormalizers: ToolResultNormalizer[], + allowLegacyViewFallback: boolean, ): Promise { const normalized = await parseAndNormalizeRequest( requestBody, @@ -843,9 +856,13 @@ async function isRequestOnlySnapshot( for (const conversation of storedData.conversations) { if ( requestMessages.length === conversation.messages.length && - requestMessages.every( - (msg, i) => - JSON.stringify(msg) === JSON.stringify(conversation.messages[i]), + requestMessages.every((_msg, i) => + messagesEqualForReplay( + requestMessages, + conversation.messages, + i, + allowLegacyViewFallback, + ), ) ) { return true; @@ -1520,11 +1537,95 @@ async function parseOpenAIResponse( } } +function messagesEqualForReplay( + requestMessages: NormalizedMessage[], + savedMessages: NormalizedMessage[], + index: number, + allowLegacyViewFallback: boolean, +): boolean { + const requestMessage = requestMessages[index]; + const savedMessage = savedMessages[index]; + const savedJson = JSON.stringify(savedMessage); + if (JSON.stringify(requestMessage) === savedJson) { + return true; + } + + if ( + !allowLegacyViewFallback || + requestMessage?.role !== "tool" || + typeof requestMessage.content !== "string" || + !requestMessage.tool_call_id + ) { + return false; + } + + let originatingToolCall: NormalizedToolCall | undefined; + for (let messageIndex = index - 1; messageIndex >= 0; messageIndex--) { + originatingToolCall = requestMessages[messageIndex].tool_calls?.find( + (toolCall) => toolCall.id === requestMessage.tool_call_id, + ); + if (originatingToolCall) { + break; + } + } + if (originatingToolCall?.function?.name !== "view") { + return false; + } + + const rawCandidate = stripLegacyViewLinePrefixes(requestMessage.content); + if (rawCandidate === undefined) { + return false; + } + + return ( + JSON.stringify({ ...requestMessage, content: rawCandidate }) === savedJson + ); +} + +function stripLegacyViewLinePrefixes(result: string): string | undefined { + const parts = result.split(/(\r\n|\n)/); + const lineIndexes: number[] = []; + + for (let index = 0; index < parts.length; index += 2) { + const isTrailingEmptyPart = + index === parts.length - 1 && parts[index] === "" && index > 0; + if (!isTrailingEmptyPart) { + lineIndexes.push(index); + } + } + + if (lineIndexes.length === 0) { + return undefined; + } + + let expectedLineNumber: number | undefined; + for (const [position, index] of lineIndexes.entries()) { + let match = /^(\d+)\. (.*)$/.exec(parts[index]); + if (!match && position === lineIndexes.length - 1) { + match = /^(\d+)\.$/.exec(parts[index]); + } + if (!match) { + return undefined; + } + + const lineNumber = Number(match[1]); + if (expectedLineNumber !== undefined && lineNumber !== expectedLineNumber) { + return undefined; + } + + expectedLineNumber = lineNumber + 1; + parts[index] = match[2] ?? ""; + } + + return parts.join("").trimEnd(); +} + // Checks if requestMessages is a prefix of savedMessages, // and returns the index of the next assistant message if found. function findAssistantIndexAfterPrefix( requestMessages: NormalizedMessage[], savedMessages: NormalizedMessage[], + allowLegacyViewFallback: boolean, ): number | undefined { const logFile = process.env.PROXY_DEBUG_LOG; const log = (msg: string) => { if (logFile) try { appendFileSync(logFile, msg + "\n"); } catch {} }; @@ -1537,7 +1638,14 @@ function findAssistantIndexAfterPrefix( for (let i = 0; i < requestMessages.length; i++) { const reqMsg = JSON.stringify(requestMessages[i]); const savedMsg = JSON.stringify(savedMessages[i]); - if (reqMsg !== savedMsg) { + if ( + !messagesEqualForReplay( + requestMessages, + savedMessages, + i, + allowLegacyViewFallback, + ) + ) { log(`mismatch at index ${i}:`); log(` REQ: ${reqMsg.substring(0, 1000)}`); log(` SAVED: ${savedMsg.substring(0, 1000)}`); diff --git a/test/snapshots/builtin_tools/should_create_a_new_file.yaml b/test/snapshots/builtin_tools/should_create_a_new_file.yaml index bf9288cf01..8afe8b38b6 100644 --- a/test/snapshots/builtin_tools/should_create_a_new_file.yaml +++ b/test/snapshots/builtin_tools/should_create_a_new_file.yaml @@ -54,6 +54,6 @@ conversations: arguments: '{"path":"${workdir}/new_file.txt"}' - role: tool tool_call_id: toolcall_2 - content: 1. Created by test + content: Created by test - role: assistant content: ✓ Done! Created `new_file.txt` with content "Created by test" and confirmed the content matches. diff --git a/test/snapshots/builtin_tools/should_edit_a_file_successfully.yaml b/test/snapshots/builtin_tools/should_edit_a_file_successfully.yaml index 0f21418628..3f4e986906 100644 --- a/test/snapshots/builtin_tools/should_edit_a_file_successfully.yaml +++ b/test/snapshots/builtin_tools/should_edit_a_file_successfully.yaml @@ -57,9 +57,8 @@ conversations: - role: tool tool_call_id: toolcall_2 content: |- - 1. Hi Universe - 2. Goodbye World - 3. + Hi Universe + Goodbye World - role: assistant content: |- Done! The file now contains: diff --git a/test/snapshots/builtin_tools/should_read_file_with_line_range.yaml b/test/snapshots/builtin_tools/should_read_file_with_line_range.yaml index cd17d86708..601ae0f04c 100644 --- a/test/snapshots/builtin_tools/should_read_file_with_line_range.yaml +++ b/test/snapshots/builtin_tools/should_read_file_with_line_range.yaml @@ -45,9 +45,9 @@ conversations: - role: tool tool_call_id: toolcall_1 content: |- - 2. line2 - 3. line3 - 4. line4 + line2 + line3 + line4 - role: assistant content: |- Lines 2 through 4 of 'lines.txt' contain: diff --git a/test/snapshots/client_options/should_use_client_cwd_for_default_workingdirectory.yaml b/test/snapshots/client_options/should_use_client_cwd_for_default_workingdirectory.yaml index 6d9167e94a..469d091288 100644 --- a/test/snapshots/client_options/should_use_client_cwd_for_default_workingdirectory.yaml +++ b/test/snapshots/client_options/should_use_client_cwd_for_default_workingdirectory.yaml @@ -25,6 +25,6 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. I am in the client cwd + content: I am in the client cwd - role: assistant content: 'The file `marker.txt` says: "I am in the client cwd"' diff --git a/test/snapshots/event_fidelity/should_emit_events_in_correct_order_for_tool_using_conversation.yaml b/test/snapshots/event_fidelity/should_emit_events_in_correct_order_for_tool_using_conversation.yaml index 8ce730f0fb..c8f272e6b9 100644 --- a/test/snapshots/event_fidelity/should_emit_events_in_correct_order_for_tool_using_conversation.yaml +++ b/test/snapshots/event_fidelity/should_emit_events_in_correct_order_for_tool_using_conversation.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Hello World + content: Hello World - role: assistant content: |- The file 'hello.txt' contains: diff --git a/test/snapshots/event_fidelity/should_emit_tool_execution_events_with_correct_fields.yaml b/test/snapshots/event_fidelity/should_emit_tool_execution_events_with_correct_fields.yaml index a6583a15ec..46fd7715ab 100644 --- a/test/snapshots/event_fidelity/should_emit_tool_execution_events_with_correct_fields.yaml +++ b/test/snapshots/event_fidelity/should_emit_tool_execution_events_with_correct_fields.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. test data + content: test data - role: assistant content: |- The file `data.txt` contains: diff --git a/test/snapshots/event_fidelity/should_preserve_message_order_in_getmessages_after_tool_use.yaml b/test/snapshots/event_fidelity/should_preserve_message_order_in_getmessages_after_tool_use.yaml index 2799cdec61..1797cc16b1 100644 --- a/test/snapshots/event_fidelity/should_preserve_message_order_in_getmessages_after_tool_use.yaml +++ b/test/snapshots/event_fidelity/should_preserve_message_order_in_getmessages_after_tool_use.yaml @@ -15,6 +15,6 @@ conversations: arguments: '{"path":"order.txt"}' - role: tool tool_call_id: toolcall_0 - content: 1. ORDER_CONTENT_42 + content: ORDER_CONTENT_42 - role: assistant content: The number in 'order.txt' is **42**. diff --git a/test/snapshots/hooks/invoke_both_hooks_for_single_tool_call.yaml b/test/snapshots/hooks/invoke_both_hooks_for_single_tool_call.yaml index 6a51857ab0..9ed9431545 100644 --- a/test/snapshots/hooks/invoke_both_hooks_for_single_tool_call.yaml +++ b/test/snapshots/hooks/invoke_both_hooks_for_single_tool_call.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Testing both hooks! + content: Testing both hooks! - role: assistant content: |- The file **both.txt** contains: diff --git a/test/snapshots/hooks/invoke_post_tool_use_hook_after_model_runs_a_tool.yaml b/test/snapshots/hooks/invoke_post_tool_use_hook_after_model_runs_a_tool.yaml index 18b324f098..2a5f1ae446 100644 --- a/test/snapshots/hooks/invoke_post_tool_use_hook_after_model_runs_a_tool.yaml +++ b/test/snapshots/hooks/invoke_post_tool_use_hook_after_model_runs_a_tool.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. World from the test! + content: World from the test! - role: assistant content: |- The file `world.txt` contains: diff --git a/test/snapshots/hooks/invoke_pre_tool_use_hook_when_model_runs_a_tool.yaml b/test/snapshots/hooks/invoke_pre_tool_use_hook_when_model_runs_a_tool.yaml index 1ce0fe67a0..f695c60f3d 100644 --- a/test/snapshots/hooks/invoke_pre_tool_use_hook_when_model_runs_a_tool.yaml +++ b/test/snapshots/hooks/invoke_pre_tool_use_hook_when_model_runs_a_tool.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Hello from the test! + content: Hello from the test! - role: assistant content: |- The file **hello.txt** contains: diff --git a/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_a_single_tool_call.yaml b/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_a_single_tool_call.yaml index 6a51857ab0..9ed9431545 100644 --- a/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_a_single_tool_call.yaml +++ b/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_a_single_tool_call.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Testing both hooks! + content: Testing both hooks! - role: assistant content: |- The file **both.txt** contains: diff --git a/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_single_tool_call.yaml b/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_single_tool_call.yaml index 6a51857ab0..9ed9431545 100644 --- a/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_single_tool_call.yaml +++ b/test/snapshots/hooks/should_invoke_both_pretooluse_and_posttooluse_hooks_for_single_tool_call.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Testing both hooks! + content: Testing both hooks! - role: assistant content: |- The file **both.txt** contains: diff --git a/test/snapshots/hooks/should_invoke_posttooluse_hook_after_model_runs_a_tool.yaml b/test/snapshots/hooks/should_invoke_posttooluse_hook_after_model_runs_a_tool.yaml index 18b324f098..2a5f1ae446 100644 --- a/test/snapshots/hooks/should_invoke_posttooluse_hook_after_model_runs_a_tool.yaml +++ b/test/snapshots/hooks/should_invoke_posttooluse_hook_after_model_runs_a_tool.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. World from the test! + content: World from the test! - role: assistant content: |- The file `world.txt` contains: diff --git a/test/snapshots/hooks/should_invoke_pretooluse_hook_when_model_runs_a_tool.yaml b/test/snapshots/hooks/should_invoke_pretooluse_hook_when_model_runs_a_tool.yaml index 1ce0fe67a0..f695c60f3d 100644 --- a/test/snapshots/hooks/should_invoke_pretooluse_hook_when_model_runs_a_tool.yaml +++ b/test/snapshots/hooks/should_invoke_pretooluse_hook_when_model_runs_a_tool.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Hello from the test! + content: Hello from the test! - role: assistant content: |- The file **hello.txt** contains: diff --git a/test/snapshots/multi_turn/should_handle_file_creation_then_reading_across_turns.yaml b/test/snapshots/multi_turn/should_handle_file_creation_then_reading_across_turns.yaml index 0d79c3e1ab..583366363a 100644 --- a/test/snapshots/multi_turn/should_handle_file_creation_then_reading_across_turns.yaml +++ b/test/snapshots/multi_turn/should_handle_file_creation_then_reading_across_turns.yaml @@ -83,7 +83,7 @@ conversations: arguments: '{"path":"${workdir}/greeting.txt"}' - role: tool tool_call_id: toolcall_2 - content: 1. Hello from multi-turn test + content: Hello from multi-turn test - role: assistant content: |- The exact contents of `greeting.txt` are: diff --git a/test/snapshots/multi_turn/should_use_tool_results_from_previous_turns.yaml b/test/snapshots/multi_turn/should_use_tool_results_from_previous_turns.yaml index b930bb46ac..96dc365c65 100644 --- a/test/snapshots/multi_turn/should_use_tool_results_from_previous_turns.yaml +++ b/test/snapshots/multi_turn/should_use_tool_results_from_previous_turns.yaml @@ -44,7 +44,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. The magic number is 42. + content: The magic number is 42. - role: assistant content: The magic number is **42**. - messages: @@ -69,7 +69,7 @@ conversations: content: Tool 'report_intent' does not exist. - role: tool tool_call_id: toolcall_1 - content: 1. The magic number is 42. + content: The magic number is 42. - role: assistant content: The magic number is **42**. - role: user diff --git a/test/snapshots/permissions/permission_handler_for_write_operations.yaml b/test/snapshots/permissions/permission_handler_for_write_operations.yaml index a4ede6fcb1..3f05a8c6de 100644 --- a/test/snapshots/permissions/permission_handler_for_write_operations.yaml +++ b/test/snapshots/permissions/permission_handler_for_write_operations.yaml @@ -47,7 +47,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. original content + content: original content - role: assistant content: "Now I'll replace 'original' with 'modified':" - role: assistant @@ -82,7 +82,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. original content + content: original content - role: assistant content: "Now I'll replace 'original' with 'modified':" tool_calls: diff --git a/test/snapshots/permissions/should_invoke_permission_handler_for_write_operations.yaml b/test/snapshots/permissions/should_invoke_permission_handler_for_write_operations.yaml index a4ede6fcb1..3f05a8c6de 100644 --- a/test/snapshots/permissions/should_invoke_permission_handler_for_write_operations.yaml +++ b/test/snapshots/permissions/should_invoke_permission_handler_for_write_operations.yaml @@ -47,7 +47,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. original content + content: original content - role: assistant content: "Now I'll replace 'original' with 'modified':" - role: assistant @@ -82,7 +82,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. original content + content: original content - role: assistant content: "Now I'll replace 'original' with 'modified':" tool_calls: diff --git a/test/snapshots/session/should_send_with_file_attachment.yaml b/test/snapshots/session/should_send_with_file_attachment.yaml index 23e05d946b..2e8e4d1d2d 100644 --- a/test/snapshots/session/should_send_with_file_attachment.yaml +++ b/test/snapshots/session/should_send_with_file_attachment.yaml @@ -58,7 +58,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. FILE_ATTACHMENT_SENTINEL + content: FILE_ATTACHMENT_SENTINEL - role: assistant content: |- The file contains: diff --git a/test/snapshots/session_config/should_accept_message_attachments.yaml b/test/snapshots/session_config/should_accept_message_attachments.yaml index e9fbabb05e..5525d1fb04 100644 --- a/test/snapshots/session_config/should_accept_message_attachments.yaml +++ b/test/snapshots/session_config/should_accept_message_attachments.yaml @@ -61,7 +61,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. This file is attached + content: This file is attached - role: assistant content: |- The attached file contains a single line of text that says: "This file is attached" diff --git a/test/snapshots/session_config/should_apply_workingdirectory_on_session_resume.yaml b/test/snapshots/session_config/should_apply_workingdirectory_on_session_resume.yaml index 52cc114f94..9d3dd78ff1 100644 --- a/test/snapshots/session_config/should_apply_workingdirectory_on_session_resume.yaml +++ b/test/snapshots/session_config/should_apply_workingdirectory_on_session_resume.yaml @@ -25,7 +25,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. I am in the resume working directory + content: I am in the resume working directory - role: assistant content: |- The file `resume-marker.txt` says: diff --git a/test/snapshots/session_config/should_use_workingdirectory_for_tool_execution.yaml b/test/snapshots/session_config/should_use_workingdirectory_for_tool_execution.yaml index 18dfab04e6..40000d491b 100644 --- a/test/snapshots/session_config/should_use_workingdirectory_for_tool_execution.yaml +++ b/test/snapshots/session_config/should_use_workingdirectory_for_tool_execution.yaml @@ -44,6 +44,6 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. I am in the subdirectory + content: I am in the subdirectory - role: assistant content: 'The file marker.txt says: "I am in the subdirectory"' diff --git a/test/snapshots/subagent_hooks/should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls.yaml b/test/snapshots/subagent_hooks/should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls.yaml index 2a73f1ef84..b1f53a99a7 100644 --- a/test/snapshots/subagent_hooks/should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls.yaml +++ b/test/snapshots/subagent_hooks/should_invoke_pretooluse_and_posttooluse_hooks_for_sub_agent_tool_calls.yaml @@ -88,7 +88,7 @@ conversations: arguments: '{"path":"${workdir}/subagent-test.txt"}' - role: tool tool_call_id: toolcall_0 - content: 1. Hello from subagent test! + content: Hello from subagent test! - role: assistant content: |- The complete contents of the file "subagent-test.txt" are: diff --git a/test/snapshots/system_message_transform/should_apply_transform_modifications_to_section_content.yaml b/test/snapshots/system_message_transform/should_apply_transform_modifications_to_section_content.yaml index 4b7c058b27..98e57919c6 100644 --- a/test/snapshots/system_message_transform/should_apply_transform_modifications_to_section_content.yaml +++ b/test/snapshots/system_message_transform/should_apply_transform_modifications_to_section_content.yaml @@ -26,7 +26,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Hello! + content: Hello! - role: assistant content: |- The file **hello.txt** contains: diff --git a/test/snapshots/system_message_transform/should_invoke_transform_callbacks_with_section_content.yaml b/test/snapshots/system_message_transform/should_invoke_transform_callbacks_with_section_content.yaml index 0b1d9755f0..c54f25e2aa 100644 --- a/test/snapshots/system_message_transform/should_invoke_transform_callbacks_with_section_content.yaml +++ b/test/snapshots/system_message_transform/should_invoke_transform_callbacks_with_section_content.yaml @@ -47,6 +47,6 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Hello transform! + content: Hello transform! - role: assistant content: 'The file `test.txt` contains: **"Hello transform!"**' diff --git a/test/snapshots/system_message_transform/should_work_with_static_overrides_and_transforms_together.yaml b/test/snapshots/system_message_transform/should_work_with_static_overrides_and_transforms_together.yaml index 0681b569dd..32d6367390 100644 --- a/test/snapshots/system_message_transform/should_work_with_static_overrides_and_transforms_together.yaml +++ b/test/snapshots/system_message_transform/should_work_with_static_overrides_and_transforms_together.yaml @@ -47,7 +47,7 @@ conversations: task. - role: tool tool_call_id: toolcall_1 - content: 1. Combo test! + content: Combo test! - role: assistant content: |- The file `combo.txt` contains: diff --git a/test/snapshots/tools/invokes_built_in_tools.yaml b/test/snapshots/tools/invokes_built_in_tools.yaml index 068cc4accf..0fba134424 100644 --- a/test/snapshots/tools/invokes_built_in_tools.yaml +++ b/test/snapshots/tools/invokes_built_in_tools.yaml @@ -15,6 +15,6 @@ conversations: arguments: '{"path":"${workdir}/README.md"}' - role: tool tool_call_id: toolcall_0 - content: "1. # ELIZA, the only chatbot you'll ever need" + content: "# ELIZA, the only chatbot you'll ever need" - role: assistant content: "The first line of README.md is: `# ELIZA, the only chatbot you'll ever need`" From cca1c288ec9357daea197b65e01a767551c26b13 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 28 Jul 2026 02:55:19 +0200 Subject: [PATCH 2/2] Handle empty legacy view output Match a legacy numbered empty-file result to the canonical content-less tool message and cover the replay edge case. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3c5c3fb3-1b45-495a-84d6-90521feaff29 --- test/harness/replayingCapiProxy.test.ts | 26 ++++++++++++++++++------- test/harness/replayingCapiProxy.ts | 10 +++++++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index a4b685ca7d..359446f159 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -727,7 +727,7 @@ Always include PINEAPPLE_COCONUT_42. async function replayToolResult( toolName: string, requestContent: string, - savedResults: Array<{ content: string; response: string }>, + savedResults: Array<{ content?: string; response: string }>, savedErrors: Array<{ content: string; status: number; @@ -736,7 +736,7 @@ Always include PINEAPPLE_COCONUT_42. ): Promise { const toolArguments = toolName === "view" ? '{"path":"file.txt"}' : "{}"; const createMessages = ( - content: string, + content: string | undefined, response?: string, ): NormalizedData["conversations"][number]["messages"] => { const messages: NormalizedData["conversations"][number]["messages"] = [ @@ -755,11 +755,15 @@ Always include PINEAPPLE_COCONUT_42. }, ], }, - { - role: "tool", - tool_call_id: "toolcall_0", - content, - }, + ...(content === undefined + ? [{ role: "tool" as const, tool_call_id: "toolcall_0" }] + : [ + { + role: "tool" as const, + tool_call_id: "toolcall_0", + content, + }, + ]), ]; if (response !== undefined) { messages.push({ role: "assistant", content: response }); @@ -931,6 +935,14 @@ Always include PINEAPPLE_COCONUT_42. expect(result).toBe("legacy view matched"); }); + test("matches a legacy numbered empty view result against omitted raw content", async () => { + const result = await replayToolResult("view", "1.", [ + { response: "empty view matched" }, + ]); + + expect(result).toBe("empty view matched"); + }); + test("prefers an exact naturally numbered view result match", async () => { const result = await replayToolResult("view", "1. alpha\n2. beta", [ { content: "alpha\nbeta", response: "legacy fallback" }, diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 1422a6d601..e67fa44eae 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -1577,9 +1577,13 @@ function messagesEqualForReplay( return false; } - return ( - JSON.stringify({ ...requestMessage, content: rawCandidate }) === savedJson - ); + const candidateMessage = { ...requestMessage }; + if (rawCandidate) { + candidateMessage.content = rawCandidate; + } else { + delete candidateMessage.content; + } + return JSON.stringify(candidateMessage) === savedJson; } function stripLegacyViewLinePrefixes(result: string): string | undefined {