Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/vscode-e2e/src/fixtures/search-files.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>)
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"nonExistentPattern12345"}',
toolCallId: "call_search_files_no_match_001",
expected: ["No results found"],
expected: ["Found 0 results."],
result: "No matches were found for `nonExistentPattern12345` in the search fixture directory.",
id: "call_search_files_no_match_002",
},
Expand Down
128 changes: 128 additions & 0 deletions apps/vscode-e2e/src/fixtures/subtasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,27 @@ const SUBTASK_FAST_CHILD_MARKER = "SUBTASK_CHILD_IMMEDIATE_COMPLETION"
const SUBTASK_XPROFILE_PARENT_MARKER = "SUBTASK_PARENT_CROSS_PROFILE"
const SUBTASK_XPROFILE_SAME_CHILD_MARKER = "SUBTASK_CHILD_SAME_PROFILE"
const SUBTASK_XPROFILE_DIFFERENT_CHILD_MARKER = "SUBTASK_CHILD_DIFFERENT_PROFILE"
const SUBTASK_FANOUT_PARENT_MARKER = "SUBTASK_PARENT_FANOUT_CONCURRENT"
const SUBTASK_FANOUT_CHILD_MARKER = "SUBTASK_CHILD_FANOUT_CONCURRENT"
const SUBTASK_FANOUT_XPROFILE_PARENT_MARKER = "SUBTASK_PARENT_FANOUT_CROSS_PROFILE"
const SUBTASK_FANOUT_XPROFILE_CHILD_MARKER = "SUBTASK_CHILD_FANOUT_CROSS_PROFILE"

const SUBTASK_CHILD_PROMPT = `${SUBTASK_CHILD_MARKER}: Ask the user exactly this follow-up question: What is the square root of 81? After the user answers, complete with only the answer.`
export const SUBTASK_PARENT_PROMPT = `${SUBTASK_PARENT_MARKER}: Use the new_task tool exactly once. Create an ask-mode subtask with this exact message: "${SUBTASK_CHILD_PROMPT}" Do not answer directly.`
export const SUBTASK_CHILD_FOLLOWUP_ANSWER = "9"
export const SUBTASK_FAST_CHILD_RESULT = "Fast child completed"
const SUBTASK_FAST_CHILD_PROMPT = `${SUBTASK_FAST_CHILD_MARKER}: Complete immediately with the exact result "${SUBTASK_FAST_CHILD_RESULT}".`
export const SUBTASK_FAST_PARENT_PROMPT = `${SUBTASK_FAST_PARENT_MARKER}: Use the new_task tool exactly once. Create an ask-mode subtask with this exact message: "${SUBTASK_FAST_CHILD_PROMPT}" Do not answer directly.`
export const SUBTASK_FANOUT_PARENT_FOLLOWUP = "Parent fan-out is still active?"
export const SUBTASK_FANOUT_CHILD_RESULT = "Fan-out child completed"
const SUBTASK_FANOUT_CHILD_PROMPT = `${SUBTASK_FANOUT_CHILD_MARKER}: Complete with the exact result "${SUBTASK_FANOUT_CHILD_RESULT}".`
export const SUBTASK_FANOUT_PARENT_PROMPT = `${SUBTASK_FANOUT_PARENT_MARKER}: Use the new_task tool exactly once. Create an ask-mode subtask with this exact message: "${SUBTASK_FANOUT_CHILD_PROMPT}" After delegation, ask the user exactly this follow-up question: ${SUBTASK_FANOUT_PARENT_FOLLOWUP}`
export const SUBTASK_FANOUT_XPROFILE_PARENT_MODEL = "openai/gpt-4.1"
export const SUBTASK_FANOUT_XPROFILE_CHILD_MODEL = "openai/gpt-4.1-mini"
export const SUBTASK_FANOUT_XPROFILE_PARENT_FOLLOWUP = "Parent cross-profile fan-out is still isolated?"
export const SUBTASK_FANOUT_XPROFILE_CHILD_RESULT = "Fan-out cross-profile child completed"
const SUBTASK_FANOUT_XPROFILE_CHILD_PROMPT = `${SUBTASK_FANOUT_XPROFILE_CHILD_MARKER}: Complete with the exact result "${SUBTASK_FANOUT_XPROFILE_CHILD_RESULT}".`
export const SUBTASK_FANOUT_XPROFILE_PARENT_PROMPT = `${SUBTASK_FANOUT_XPROFILE_PARENT_MARKER}: Use the new_task tool exactly once. Create an ask-mode subtask with this exact message: "${SUBTASK_FANOUT_XPROFILE_CHILD_PROMPT}" After delegation, ask the user exactly this follow-up question: ${SUBTASK_FANOUT_XPROFILE_PARENT_FOLLOWUP}`

const SUBTASK_INTERRUPT_CHILD_PROMPT = `${SUBTASK_INTERRUPT_CHILD_MARKER}: Ask the user exactly this follow-up question: What is the square root of 81? After the user answers, complete with only the answer.`
export const SUBTASK_INTERRUPT_PARENT_PROMPT = `${SUBTASK_INTERRUPT_PARENT_MARKER}: Use the new_task tool exactly once. Create an ask-mode subtask with this exact message: "${SUBTASK_INTERRUPT_CHILD_PROMPT}" Do not answer directly. When the subtask returns, complete with the exact result "Interrupted parent resumed".`
Expand Down Expand Up @@ -135,6 +149,120 @@ export function addSubtaskFixtures(mock: InstanceType<typeof LLMock>) {
},
})

mock.addFixture({
match: {
userMessage: new RegExp(SUBTASK_FANOUT_PARENT_MARKER),
sequenceIndex: 0,
},
response: {
toolCalls: [
{
name: "new_task",
arguments: JSON.stringify({
mode: "ask",
message: SUBTASK_FANOUT_CHILD_PROMPT,
}),
id: "call_subtasks_fanout_parent_new_task_001",
},
],
},
})

mock.addFixture({
match: {
predicate: (req: ChatCompletionRequest) =>
lastUserMessageContains(req, SUBTASK_FANOUT_CHILD_MARKER) &&
!requestContains(req, [SUBTASK_FANOUT_PARENT_MARKER]),
},
latency: 15_000,
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: SUBTASK_FANOUT_CHILD_RESULT }),
id: "call_subtasks_fanout_child_completion_002",
},
],
},
})

mock.addFixture({
match: {
predicate: (req: ChatCompletionRequest) =>
requestContains(req, [SUBTASK_FANOUT_PARENT_MARKER, "Delegated to child task"]) &&
!requestContains(req, [SUBTASK_RESULT_INJECTION]),
},
response: {
toolCalls: [
{
name: "ask_followup_question",
arguments: JSON.stringify({
question: SUBTASK_FANOUT_PARENT_FOLLOWUP,
follow_up: [{ text: "continue" }],
}),
id: "call_subtasks_fanout_parent_followup_003",
},
],
},
})

mock.addFixture({
match: {
userMessage: new RegExp(SUBTASK_FANOUT_XPROFILE_PARENT_MARKER),
sequenceIndex: 0,
},
response: {
toolCalls: [
{
name: "new_task",
arguments: JSON.stringify({
mode: "ask",
message: SUBTASK_FANOUT_XPROFILE_CHILD_PROMPT,
}),
id: "call_subtasks_fanout_xprofile_parent_new_task_001",
},
],
},
})

mock.addFixture({
match: {
predicate: (req: ChatCompletionRequest) =>
lastUserMessageContains(req, SUBTASK_FANOUT_XPROFILE_CHILD_MARKER) &&
!requestContains(req, [SUBTASK_FANOUT_XPROFILE_PARENT_MARKER]),
},
latency: 15_000,
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: SUBTASK_FANOUT_XPROFILE_CHILD_RESULT }),
id: "call_subtasks_fanout_xprofile_child_completion_002",
},
],
},
})

mock.addFixture({
match: {
predicate: (req: ChatCompletionRequest) =>
requestContains(req, [SUBTASK_FANOUT_XPROFILE_PARENT_MARKER, "Delegated to child task"]) &&
!requestContains(req, [SUBTASK_RESULT_INJECTION]),
},
response: {
toolCalls: [
{
name: "ask_followup_question",
arguments: JSON.stringify({
question: SUBTASK_FANOUT_XPROFILE_PARENT_FOLLOWUP,
follow_up: [{ text: "continue" }],
}),
id: "call_subtasks_fanout_xprofile_parent_followup_003",
},
],
},
})

// The parent prompt embeds SUBTASK_FAST_CHILD_MARKER verbatim, so parent-resume turns
// can also match a bare substring check (same collision class as #561). Exclude the
// parent marker so those turns fall through to the parent-resume fixture below.
Expand Down
25 changes: 22 additions & 3 deletions apps/vscode-e2e/src/fixtures/terminal-reuse-shell-race.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,19 @@ import { toolResultContains } from "./tool-result"

export function addTerminalReuseShellRaceFixtures(mock: InstanceType<typeof LLMock>) {
// First command completes — model issues a second command on the same terminal.
// With the temp-script fix, both commands now deliver real output.
mock.addFixture({
match: {
predicate: (req) => toolResultContains(req, "call_terminal_reuse_001", ["first", "Exit code: 0"]),
predicate: (req) => {
const messages = Array.isArray(req?.messages) ? req.messages : []
const lastToolMsg = messages.filter((message) => message?.role === "tool").at(-1)

return (
lastToolMsg?.tool_call_id === "call_terminal_reuse_001" &&
toolResultContains(req, "call_terminal_reuse_001", [
"Command was submitted in the VS Code terminal",
])
)
},
},
response: {
toolCalls: [
Expand All @@ -25,7 +34,17 @@ export function addTerminalReuseShellRaceFixtures(mock: InstanceType<typeof LLMo
// Second command on the reused terminal also completes.
mock.addFixture({
match: {
predicate: (req) => toolResultContains(req, "call_terminal_reuse_002", ["second", "Exit code: 0"]),
predicate: (req) => {
const messages = Array.isArray(req?.messages) ? req.messages : []
const lastToolMsg = messages.filter((message) => message?.role === "tool").at(-1)

return (
lastToolMsg?.tool_call_id === "call_terminal_reuse_002" &&
toolResultContains(req, "call_terminal_reuse_002", [
"Command was submitted in the VS Code terminal",
])
)
},
},
response: {
toolCalls: [
Expand Down
Loading
Loading