From aa64f781c5fb59347ebfbce3278c6b9a8b50141e Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 00:33:09 +0800 Subject: [PATCH 1/9] test(vscode-e2e): cover orchestrator fan-out delegation --- apps/vscode-e2e/package.json | 4 +- .../src/fixtures/orchestrator-plan.spec.ts | 101 +++++++++++++++++ .../src/fixtures/orchestrator-plan.ts | 73 +++++++++++++ apps/vscode-e2e/src/fixtures/orchestrator.ts | 96 ++++++++++++++++ apps/vscode-e2e/src/runTest.ts | 2 + .../vscode-e2e/src/suite/orchestrator.test.ts | 103 ++++++++++++++++++ apps/vscode-e2e/vitest.config.ts | 8 ++ pnpm-lock.yaml | 3 + 8 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts create mode 100644 apps/vscode-e2e/src/fixtures/orchestrator-plan.ts create mode 100644 apps/vscode-e2e/src/fixtures/orchestrator.ts create mode 100644 apps/vscode-e2e/src/suite/orchestrator.test.ts create mode 100644 apps/vscode-e2e/vitest.config.ts diff --git a/apps/vscode-e2e/package.json b/apps/vscode-e2e/package.json index 3b1a3fb18f..c5b44786a8 100644 --- a/apps/vscode-e2e/package.json +++ b/apps/vscode-e2e/package.json @@ -5,6 +5,7 @@ "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc -p tsconfig.esm.json --noEmit", "format": "prettier --write src", + "test:unit": "vitest run --config vitest.config.ts", "test:ci": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && pnpm test:run", "test:ci:mock": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && USE_MOCK=true pnpm test:run", "test:record": "AIMOCK_RECORD=true pnpm test:ci", @@ -23,6 +24,7 @@ "dotenv-cli": "11.0.0", "glob": "11.1.0", "mocha": "11.2.2", - "rimraf": "6.0.1" + "rimraf": "6.0.1", + "vitest": "4.1.9" } } diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts new file mode 100644 index 0000000000..ab31c905fc --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest" + +import { + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + buildOrchestratorResumeExpectations, + shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorResumeRequest, +} from "./orchestrator-plan" + +describe("orchestrator fan-out delegation plan", () => { + it("defines the first commit single-round ask/architect/code child sequence", () => { + expect(ORCHESTRATOR_FAN_OUT_PARENT_PROMPT).toContain("ORCHESTRATOR_SINGLE_ROUND_FAN_OUT") + expect(ORCHESTRATOR_FAN_OUT_CHILD_STEPS).toEqual([ + expect.objectContaining({ + mode: "ask", + summary: "Requirement summary: gather requirements for the reporting workflow.", + }), + expect.objectContaining({ + mode: "architect", + summary: "Design summary: outline a minimal fan-in architecture.", + }), + expect.objectContaining({ + mode: "code", + summary: "Implementation summary: implement the delegated workflow skeleton.", + }), + ]) + }) + + it("builds cumulative parent resume expectations for fan-in matching", () => { + expect(buildOrchestratorResumeExpectations()).toEqual([ + { + stepIndex: 1, + requiredSummaries: ["Requirement summary: gather requirements for the reporting workflow."], + nextMode: "architect", + }, + { + stepIndex: 2, + requiredSummaries: [ + "Requirement summary: gather requirements for the reporting workflow.", + "Design summary: outline a minimal fan-in architecture.", + ], + nextMode: "code", + }, + { + stepIndex: 3, + requiredSummaries: [ + "Requirement summary: gather requirements for the reporting workflow.", + "Design summary: outline a minimal fan-in architecture.", + "Implementation summary: implement the delegated workflow skeleton.", + ], + nextMode: undefined, + }, + ]) + }) + + it("matches child requests without matching the parent prompt that embeds child markers", () => { + expect( + shouldMatchOrchestratorChildRequest( + "child ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + ), + ).toBe(true) + expect( + shouldMatchOrchestratorChildRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT embeds ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + ), + ).toBe(false) + }) + + it("matches parent resume requests only after cumulative child result injection", () => { + const [firstExpectation] = buildOrchestratorResumeExpectations() + const resumeRequest = `ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\n${firstExpectation.requiredSummaries[0]}` + + expect(shouldMatchOrchestratorResumeRequest(resumeRequest, firstExpectation.requiredSummaries)).toBe(true) + expect( + shouldMatchOrchestratorResumeRequest( + `ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\nmissing ${firstExpectation.requiredSummaries[0]}`, + firstExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorResumeRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT without result", + firstExpectation.requiredSummaries, + ), + ).toBe(false) + }) + + it("keeps the final parent result fan-in explicit and reviewable", () => { + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain( + "Requirement summary: gather requirements for the reporting workflow.", + ) + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain("Design summary: outline a minimal fan-in architecture.") + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain( + "Implementation summary: implement the delegated workflow skeleton.", + ) + }) +}) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts new file mode 100644 index 0000000000..46f69cdbe3 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -0,0 +1,73 @@ +export type OrchestratorFanOutMode = "ask" | "architect" | "code" + +export type OrchestratorFanOutChildStep = { + readonly mode: OrchestratorFanOutMode + readonly marker: string + readonly prompt: string + readonly summary: string + readonly newTaskToolCallId: string + readonly completionToolCallId: string +} + +export type OrchestratorResumeExpectation = { + readonly stepIndex: number + readonly requiredSummaries: string[] + readonly nextMode?: OrchestratorFanOutMode +} + +export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT" +export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:" + +export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [ + { + mode: "ask", + marker: "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD: Complete immediately with the exact result "Requirement summary: gather requirements for the reporting workflow."', + summary: "Requirement summary: gather requirements for the reporting workflow.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_001", + completionToolCallId: "call_orchestrator_fan_out_requirements_completion_001", + }, + { + mode: "architect", + marker: "ORCHESTRATOR_SINGLE_ROUND_DESIGN_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_DESIGN_CHILD: Complete immediately with the exact result "Design summary: outline a minimal fan-in architecture."', + summary: "Design summary: outline a minimal fan-in architecture.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_002", + completionToolCallId: "call_orchestrator_fan_out_design_completion_001", + }, + { + mode: "code", + marker: "ORCHESTRATOR_SINGLE_ROUND_IMPLEMENTATION_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_IMPLEMENTATION_CHILD: Complete immediately with the exact result "Implementation summary: implement the delegated workflow skeleton."', + summary: "Implementation summary: implement the delegated workflow skeleton.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_003", + completionToolCallId: "call_orchestrator_fan_out_implementation_completion_001", + }, +] + +export const ORCHESTRATOR_FAN_OUT_PARENT_PROMPT = `${ORCHESTRATOR_FAN_OUT_MARKER}: Delegate exactly three children in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].prompt}". After it returns, create an architect-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1].prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2].prompt}". After all three return, complete with the final fan-in summary containing every child summary.` + +export const ORCHESTRATOR_FAN_OUT_FINAL_RESULT = `Orchestrator fan-in complete:\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1].summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2].summary}` + +export function buildOrchestratorResumeExpectations(): OrchestratorResumeExpectation[] { + return ORCHESTRATOR_FAN_OUT_CHILD_STEPS.map((_step, index) => ({ + stepIndex: index + 1, + requiredSummaries: ORCHESTRATOR_FAN_OUT_CHILD_STEPS.slice(0, index + 1).map(({ summary }) => summary), + nextMode: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[index + 1]?.mode, + })) +} + +export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean { + return rawRequest.includes(childMarker) && !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) +} + +export function shouldMatchOrchestratorResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => rawRequest.includes(`Result:\\n${summary}`)) + ) +} diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts new file mode 100644 index 0000000000..f8ac225e57 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -0,0 +1,96 @@ +import { LLMock } from "@copilotkit/aimock" +import type { ChatCompletionRequest } from "@copilotkit/aimock" + +import { + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_MARKER, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + buildOrchestratorResumeExpectations, + shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorResumeRequest, +} from "./orchestrator-plan" + +export { + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_MARKER, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + buildOrchestratorResumeExpectations, + shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorResumeRequest, +} + +const requestText = (req: ChatCompletionRequest) => JSON.stringify(req) + +export function addOrchestratorFixtures(mock: InstanceType) { + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_FAN_OUT_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].mode, + message: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].prompt, + }), + id: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].newTaskToolCallId, + }, + ], + }, + }) + + for (const step of ORCHESTRATOR_FAN_OUT_CHILD_STEPS) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorChildRequest(requestText(req), step.marker), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: step.summary }), + id: step.completionToolCallId, + }, + ], + }, + }) + } + + for (const expectation of [...buildOrchestratorResumeExpectations()].reverse()) { + const nextStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[expectation.stepIndex] + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorResumeRequest(requestText(req), expectation.requiredSummaries), + }, + response: { + toolCalls: nextStep + ? [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: nextStep.mode, + message: nextStep.prompt, + }), + id: nextStep.newTaskToolCallId, + }, + ] + : [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_FAN_OUT_FINAL_RESULT }), + id: "call_orchestrator_fan_out_parent_completion_004", + }, + ], + }, + }) + } +} diff --git a/apps/vscode-e2e/src/runTest.ts b/apps/vscode-e2e/src/runTest.ts index 482e73e945..bbaab7712a 100644 --- a/apps/vscode-e2e/src/runTest.ts +++ b/apps/vscode-e2e/src/runTest.ts @@ -19,6 +19,7 @@ import { addListFilesResultFixtures } from "./fixtures/list-files" import { addReadFileResultFixtures } from "./fixtures/read-file" import { addSearchFilesResultFixtures } from "./fixtures/search-files" import { addSubtaskFixtures } from "./fixtures/subtasks" +import { addOrchestratorFixtures } from "./fixtures/orchestrator" import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool" import { addWriteToFileResultFixtures } from "./fixtures/write-to-file" @@ -126,6 +127,7 @@ async function main() { addReadFileResultFixtures(mock) addSearchFilesResultFixtures(mock) addSubtaskFixtures(mock) + addOrchestratorFixtures(mock) addUseMcpToolResultFixtures(mock) addWriteToFileResultFixtures(mock) addDeepSeekV4Fixtures(mock) diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts new file mode 100644 index 0000000000..4a097c1d88 --- /dev/null +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -0,0 +1,103 @@ +import * as assert from "assert" + +import { RooCodeEventName, type ClineMessage } from "@roo-code/types" + +import { setDefaultSuiteTimeout } from "./test-utils" +import { sleep, waitFor, waitUntilCompleted } from "./utils" +import { + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, +} from "../fixtures/orchestrator" + +suite("Roo Code Orchestrator", function () { + setDefaultSuiteTimeout(this) + + test("orchestrator parent fans out once and fans in three delegated child summaries", async () => { + const api = globalThis.api + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === ORCHESTRATOR_FAN_OUT_CHILD_STEPS.length) + + const childIds = delegationCompletions.map(({ childId }) => childId) + assert.strictEqual(new Set(childIds).size, 3, "Orchestrator should create three distinct child tasks") + assert.deepStrictEqual( + delegationCompletions.map(({ parentId }) => parentId), + [parentTaskId, parentTaskId, parentTaskId], + "Every delegation completed event should point to the orchestrator parent", + ) + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + ORCHESTRATOR_FAN_OUT_CHILD_STEPS.map(({ summary }) => summary), + "Delegation completed events should preserve each child summary in order", + ) + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Parent history item should exist") + assert.strictEqual(parent.status, "completed", "Parent should be completed after final fan-in") + assert.deepStrictEqual(parent.childIds, childIds, "Parent childIds should contain all three children") + + for (const [index, childId] of childIds.entries()) { + const child = await api.getTaskHistoryItem(childId) + assert.ok(child, `Child ${index + 1} history item should exist`) + assert.strictEqual(child.parentTaskId, parentTaskId, `Child ${index + 1} should point back to parent`) + assert.strictEqual(child.status, "completed", `Child ${index + 1} should be completed`) + assert.strictEqual( + child.completionResultSummary, + ORCHESTRATOR_FAN_OUT_CHILD_STEPS[index]!.summary, + `Child ${index + 1} completion summary should be persisted`, + ) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => !!text) + + assert.strictEqual(parentCompletionText, ORCHESTRATOR_FAN_OUT_FINAL_RESULT) + for (const { summary } of ORCHESTRATOR_FAN_OUT_CHILD_STEPS) { + assert.ok(parentCompletionText.includes(summary), `Final parent completion should include ${summary}`) + } + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + while (api.getCurrentTaskStack().length > 0) { + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after cleanup") + await sleep(100) + } + }) +}) diff --git a/apps/vscode-e2e/vitest.config.ts b/apps/vscode-e2e/vitest.config.ts new file mode 100644 index 0000000000..d6ecaf0e42 --- /dev/null +++ b/apps/vscode-e2e/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + watch: false, + include: ["src/**/*.spec.ts"], + }, +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c3dd070ac..1e7aa854fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,9 @@ importers: rimraf: specifier: 6.0.1 version: 6.0.1 + vitest: + specifier: 4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@22.20.1)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@26.1.0)(vite@8.1.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) apps/vscode-nightly: devDependencies: From 48f5fba51077cd376fd4151b3d57ba0f091d8b18 Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 00:42:30 +0800 Subject: [PATCH 2/9] test(vscode-e2e): stress orchestrator repeated delegation --- .../src/fixtures/orchestrator-plan.spec.ts | 81 +++++++++++- .../src/fixtures/orchestrator-plan.ts | 103 ++++++++++++++- apps/vscode-e2e/src/fixtures/orchestrator.ts | 88 ++++++++++++- .../vscode-e2e/src/suite/orchestrator.test.ts | 120 ++++++++++++++++++ 4 files changed, 381 insertions(+), 11 deletions(-) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts index ab31c905fc..4b5d6e3e8c 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -4,8 +4,14 @@ import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_MARKER, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" @@ -71,7 +77,7 @@ describe("orchestrator fan-out delegation plan", () => { }) it("matches parent resume requests only after cumulative child result injection", () => { - const [firstExpectation] = buildOrchestratorResumeExpectations() + const firstExpectation = buildOrchestratorResumeExpectations()[0]! const resumeRequest = `ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\n${firstExpectation.requiredSummaries[0]}` expect(shouldMatchOrchestratorResumeRequest(resumeRequest, firstExpectation.requiredSummaries)).toBe(true) @@ -98,4 +104,77 @@ describe("orchestrator fan-out delegation plan", () => { "Implementation summary: implement the delegated workflow skeleton.", ) }) + + it("defines a three-round repeated delegation sequence with ask/architect/code children per round", () => { + expect(ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT).toContain(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS).toHaveLength(9) + expect( + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, mode }) => ({ round, role, mode })), + ).toEqual([ + { round: 1, role: "requirements", mode: "ask" }, + { round: 1, role: "design", mode: "architect" }, + { round: 1, role: "implementation", mode: "code" }, + { round: 2, role: "requirements", mode: "ask" }, + { round: 2, role: "design", mode: "architect" }, + { round: 2, role: "implementation", mode: "code" }, + { round: 3, role: "requirements", mode: "ask" }, + { round: 3, role: "design", mode: "architect" }, + { round: 3, role: "implementation", mode: "code" }, + ]) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]?.summary).toBe( + "Round 1 requirements summary: capture reporting workflow constraints.", + ) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[8]?.summary).toBe( + "Round 3 implementation summary: validate repeated delegation convergence.", + ) + }) + + it("builds cumulative repeated delegation resume expectations across all rounds", () => { + const expectations = buildOrchestratorRepeatedResumeExpectations() + + expect(expectations).toHaveLength(9) + expect(expectations[0]).toEqual({ + stepIndex: 1, + requiredSummaries: ["Round 1 requirements summary: capture reporting workflow constraints."], + nextMode: "architect", + }) + expect(expectations[2]).toEqual({ + stepIndex: 3, + requiredSummaries: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.slice(0, 3).map(({ summary }) => summary), + nextMode: "ask", + }) + expect(expectations[8]).toEqual({ + stepIndex: 9, + requiredSummaries: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ summary }) => summary), + nextMode: undefined, + }) + }) + + it("matches repeated delegation parent resumes only for cumulative child result injection", () => { + const expectations = buildOrchestratorRepeatedResumeExpectations() + const thirdExpectation = expectations[2]! + const resumeRequest = `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${thirdExpectation.requiredSummaries.join(" completed.\\n\\nResult:\\n")}` + + expect(shouldMatchOrchestratorRepeatedResumeRequest(resumeRequest, thirdExpectation.requiredSummaries)).toBe( + true, + ) + expect( + shouldMatchOrchestratorRepeatedResumeRequest( + `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER} completed.\\n\\nResult:\\nmissing ${thirdExpectation.requiredSummaries[2]}`, + thirdExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorRepeatedResumeRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\nRound 1 requirements summary: capture reporting workflow constraints.", + ["Round 1 requirements summary: capture reporting workflow constraints."], + ), + ).toBe(false) + }) + + it("composes a repeated delegation final result with every round and child summary", () => { + for (const { round, role, summary } of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { + expect(ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT).toContain(`Round ${round} ${role}: ${summary}`) + } + }) }) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts index 46f69cdbe3..062e1edb35 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -9,6 +9,13 @@ export type OrchestratorFanOutChildStep = { readonly completionToolCallId: string } +export type OrchestratorRepeatedDelegationRole = "requirements" | "design" | "implementation" + +export type OrchestratorRepeatedDelegationChildStep = OrchestratorFanOutChildStep & { + readonly round: number + readonly role: OrchestratorRepeatedDelegationRole +} + export type OrchestratorResumeExpectation = { readonly stepIndex: number readonly requiredSummaries: string[] @@ -17,6 +24,7 @@ export type OrchestratorResumeExpectation = { export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT" export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:" +export const ORCHESTRATOR_REPEATED_DELEGATION_MARKER = "ORCHESTRATOR_REPEATED_DELEGATION_STRESS" export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [ { @@ -45,20 +53,90 @@ export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildS }, ] -export const ORCHESTRATOR_FAN_OUT_PARENT_PROMPT = `${ORCHESTRATOR_FAN_OUT_MARKER}: Delegate exactly three children in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].prompt}". After it returns, create an architect-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1].prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2].prompt}". After all three return, complete with the final fan-in summary containing every child summary.` +export const ORCHESTRATOR_FAN_OUT_PARENT_PROMPT = `${ORCHESTRATOR_FAN_OUT_MARKER}: Delegate exactly three children in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]!.prompt}". After it returns, create an architect-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1]!.prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2]!.prompt}". After all three return, complete with the final fan-in summary containing every child summary.` -export const ORCHESTRATOR_FAN_OUT_FINAL_RESULT = `Orchestrator fan-in complete:\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1].summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2].summary}` +export const ORCHESTRATOR_FAN_OUT_FINAL_RESULT = `Orchestrator fan-in complete:\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]!.summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1]!.summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2]!.summary}` -export function buildOrchestratorResumeExpectations(): OrchestratorResumeExpectation[] { - return ORCHESTRATOR_FAN_OUT_CHILD_STEPS.map((_step, index) => ({ +const ORCHESTRATOR_REPEATED_DELEGATION_ROLES: ReadonlyArray<{ + readonly role: OrchestratorRepeatedDelegationRole + readonly mode: OrchestratorFanOutMode + readonly summaryByRound: readonly [string, string, string] +}> = [ + { + role: "requirements", + mode: "ask", + summaryByRound: [ + "capture reporting workflow constraints.", + "confirm dashboard stakeholder needs.", + "verify rollout acceptance criteria.", + ], + }, + { + role: "design", + mode: "architect", + summaryByRound: [ + "outline fan-in checkpoints.", + "refine retry-safe orchestration boundaries.", + "document final aggregation shape.", + ], + }, + { + role: "implementation", + mode: "code", + summaryByRound: [ + "stub delegated workflow shell.", + "wire summary collection fixtures.", + "validate repeated delegation convergence.", + ], + }, +] + +export const ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS: readonly OrchestratorRepeatedDelegationChildStep[] = ( + [1, 2, 3] as const +).flatMap((round, roundIndex) => + ORCHESTRATOR_REPEATED_DELEGATION_ROLES.map(({ role, mode, summaryByRound }, roleIndex) => { + const summary = `Round ${round} ${role} summary: ${summaryByRound[roundIndex]!}` + const marker = `ORCHESTRATOR_REPEATED_ROUND_${round}_${role.toUpperCase()}_CHILD` + + return { + round, + role, + mode, + marker, + prompt: `${marker}: Complete immediately with the exact result "${summary}"`, + summary, + newTaskToolCallId: `call_orchestrator_repeated_parent_new_task_${String((round - 1) * 3 + roleIndex + 1).padStart(3, "0")}`, + completionToolCallId: `call_orchestrator_repeated_${role}_round_${round}_completion_001`, + } + }), +) + +export const ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER}: Run exactly three rounds. In each round, delegate exactly three children in order: ask-mode requirements, architect-mode design, then code-mode implementation. Use these exact child messages in order: ${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ prompt }) => `"${prompt}"`).join("; ")}. After each child returns, resume the parent before creating the next child. After all nine children return, complete with a final summary containing every round and child summary.` + +export const ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT = `Orchestrator repeated delegation complete:\n${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, summary }) => `- Round ${round} ${role}: ${summary}`).join("\\n")}` + +function buildResumeExpectations(steps: readonly OrchestratorFanOutChildStep[]): OrchestratorResumeExpectation[] { + return steps.map((_step, index) => ({ stepIndex: index + 1, - requiredSummaries: ORCHESTRATOR_FAN_OUT_CHILD_STEPS.slice(0, index + 1).map(({ summary }) => summary), - nextMode: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[index + 1]?.mode, + requiredSummaries: steps.slice(0, index + 1).map(({ summary }) => summary), + nextMode: steps[index + 1]?.mode, })) } +export function buildOrchestratorResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_FAN_OUT_CHILD_STEPS) +} + +export function buildOrchestratorRepeatedResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) +} + export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean { - return rawRequest.includes(childMarker) && !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) + return ( + rawRequest.includes(childMarker) && + !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) + ) } export function shouldMatchOrchestratorResumeRequest( @@ -71,3 +149,14 @@ export function shouldMatchOrchestratorResumeRequest( requiredSummaries.every((summary) => rawRequest.includes(`Result:\\n${summary}`)) ) } + +export function shouldMatchOrchestratorRepeatedResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => rawRequest.includes(`Result:\\n${summary}`)) + ) +} diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts index f8ac225e57..ba3694c9f8 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -7,8 +7,14 @@ import { ORCHESTRATOR_FAN_OUT_MARKER, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_MARKER, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" @@ -18,14 +24,23 @@ export { ORCHESTRATOR_FAN_OUT_MARKER, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_MARKER, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } const requestText = (req: ChatCompletionRequest) => JSON.stringify(req) export function addOrchestratorFixtures(mock: InstanceType) { + const firstFanOutStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]! + const firstRepeatedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]! + mock.addFixture({ match: { userMessage: new RegExp(ORCHESTRATOR_FAN_OUT_MARKER), @@ -36,10 +51,10 @@ export function addOrchestratorFixtures(mock: InstanceType) { { name: "new_task", arguments: JSON.stringify({ - mode: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].mode, - message: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].prompt, + mode: firstFanOutStep.mode, + message: firstFanOutStep.prompt, }), - id: ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0].newTaskToolCallId, + id: firstFanOutStep.newTaskToolCallId, }, ], }, @@ -93,4 +108,71 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) } + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_REPEATED_DELEGATION_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstRepeatedStep.mode, + message: firstRepeatedStep.prompt, + }), + id: firstRepeatedStep.newTaskToolCallId, + }, + ], + }, + }) + + for (const step of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorChildRequest(requestText(req), step.marker), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: step.summary }), + id: step.completionToolCallId, + }, + ], + }, + }) + } + + for (const expectation of [...buildOrchestratorRepeatedResumeExpectations()].reverse()) { + const nextStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[expectation.stepIndex] + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorRepeatedResumeRequest(requestText(req), expectation.requiredSummaries), + }, + response: { + toolCalls: nextStep + ? [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: nextStep.mode, + message: nextStep.prompt, + }), + id: nextStep.newTaskToolCallId, + }, + ] + : [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT }), + id: "call_orchestrator_repeated_parent_completion_010", + }, + ], + }, + }) + } } diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts index 4a097c1d88..cf6b9a9fd2 100644 --- a/apps/vscode-e2e/src/suite/orchestrator.test.ts +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -8,6 +8,9 @@ import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, } from "../fixtures/orchestrator" suite("Roo Code Orchestrator", function () { @@ -100,4 +103,121 @@ suite("Roo Code Orchestrator", function () { await sleep(100) } }) + + test("orchestrator parent repeats three rounds of delegated child fan-in without stack duplication", async () => { + const api = globalThis.api + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + const parentStackSnapshots: string[][] = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + parentStackSnapshots.push(api.getCurrentTaskStack()) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.length) + + const childIds = delegationCompletions.map(({ childId }) => childId) + assert.strictEqual(childIds.length, 9, "Repeated delegation should create nine child tasks") + assert.strictEqual(new Set(childIds).size, 9, "Repeated delegation should create nine distinct child tasks") + assert.deepStrictEqual( + delegationCompletions.map(({ parentId }) => parentId), + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(() => parentTaskId), + "Every repeated delegation completed event should point to the orchestrator parent", + ) + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ summary }) => summary), + "Repeated delegation completed events should preserve every round summary in order", + ) + + for (const [index, snapshot] of parentStackSnapshots.entries()) { + const parentOccurrences = snapshot.filter((taskId) => taskId === parentTaskId).length + assert.ok( + parentOccurrences <= 1, + `Parent should not be duplicated in stack after child ${index + 1} resumes`, + ) + } + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Repeated delegation parent history item should exist") + assert.strictEqual( + parent.status, + "completed", + "Repeated delegation parent should complete after final fan-in", + ) + assert.deepStrictEqual( + parent.childIds, + childIds, + "Parent childIds should contain all nine children in order", + ) + assert.strictEqual(new Set(parent.childIds ?? []).size, 9, "Parent childIds should not contain duplicates") + + for (const [index, childId] of childIds.entries()) { + const expectedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[index]! + const child = await api.getTaskHistoryItem(childId) + assert.ok(child, `Repeated delegation child ${index + 1} history item should exist`) + assert.strictEqual(child.parentTaskId, parentTaskId, `Child ${index + 1} should point back to parent`) + assert.strictEqual(child.status, "completed", `Child ${index + 1} should be completed`) + assert.strictEqual( + child.mode, + expectedStep.mode, + `Child ${index + 1} mode should match the repeated plan`, + ) + assert.strictEqual( + child.completionResultSummary, + expectedStep.summary, + `Child ${index + 1} round ${expectedStep.round} ${expectedStep.role} summary should be persisted`, + ) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => !!text) + + assert.strictEqual(parentCompletionText, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT) + for (const { round, role, summary } of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { + assert.ok( + parentCompletionText.includes(`Round ${round} ${role}: ${summary}`), + `Final parent completion should include round ${round} ${role} summary`, + ) + } + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + while (api.getCurrentTaskStack().length > 0) { + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after cleanup") + await sleep(100) + } + }) }) From d7e1e0fb183520089b712891540cae0036562d16 Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 01:00:10 +0800 Subject: [PATCH 3/9] test(api): expose task resource diagnostics --- packages/types/src/api.ts | 20 ++++ .../api-resource-diagnostics.spec.ts | 111 ++++++++++++++++++ src/extension/api.ts | 29 +++++ 3 files changed, 160 insertions(+) create mode 100644 src/extension/__tests__/api-resource-diagnostics.spec.ts diff --git a/packages/types/src/api.ts b/packages/types/src/api.ts index 89e9c8bc2b..fb9f77b2d5 100644 --- a/packages/types/src/api.ts +++ b/packages/types/src/api.ts @@ -9,6 +9,22 @@ import type { IpcMessage, IpcServerEvents } from "./ipc.js" export type RooCodeAPIEvents = RooCodeEvents +export type RooCodeResourceDiagnosticEventName = + | "message" + | "taskCreated" + | "taskStarted" + | "taskCompleted" + | "taskAborted" + | "taskDelegationCompleted" + | "taskDelegationResumed" + | "taskModeSwitched" + +export interface RooCodeResourceDiagnostics { + registeredTaskCount: number + currentTaskStackLength: number + listenerCounts: Record +} + export interface RooCodeAPI extends EventEmitter { /** * Starts a new task with an optional initial message and images. @@ -56,6 +72,10 @@ export interface RooCodeAPI extends EventEmitter { * @returns An array of task IDs. */ getCurrentTaskStack(): string[] + /** + * Returns stable task resource counters for test diagnostics. + */ + getResourceDiagnostics(): RooCodeResourceDiagnostics /** * Clears the current task. */ diff --git a/src/extension/__tests__/api-resource-diagnostics.spec.ts b/src/extension/__tests__/api-resource-diagnostics.spec.ts new file mode 100644 index 0000000000..7fbec9d976 --- /dev/null +++ b/src/extension/__tests__/api-resource-diagnostics.spec.ts @@ -0,0 +1,111 @@ +import { EventEmitter } from "events" + +import { describe, expect, it, vi, beforeEach, type Mock } from "vitest" +import * as vscode from "vscode" + +import { RooCodeEventName } from "@roo-code/types" + +import { API } from "../api" +import type { ClineProvider } from "../../core/webview/ClineProvider" + +type ProviderDouble = EventEmitter & { + context: vscode.ExtensionContext + getCurrentTaskStack: Mock<() => string[]> + on: EventEmitter["on"] +} + +type TaskDouble = EventEmitter & { + taskId: string + parentTaskId?: string + approveAsk: Mock<() => void> + handleWebviewAskResponse: Mock<(response: "messageResponse", text?: string, images?: string[]) => void> +} + +function asClineProvider(provider: ProviderDouble): ClineProvider { + // ClineProvider has private members, so a structural test double requires an unknown bridge. + return provider as unknown as ClineProvider +} + +function createProvider(): ProviderDouble { + const provider = new EventEmitter() as ProviderDouble + provider.context = {} as vscode.ExtensionContext + provider.getCurrentTaskStack = vi.fn().mockReturnValue([]) + return provider +} + +function createTask(taskId: string): TaskDouble { + const task = new EventEmitter() as TaskDouble + task.taskId = taskId + task.approveAsk = vi.fn() + task.handleWebviewAskResponse = vi.fn() + return task +} + +describe("API#getResourceDiagnostics", () => { + let api: API + let sidebarProvider: ProviderDouble + + beforeEach(() => { + sidebarProvider = createProvider() + api = new API({ appendLine: vi.fn() } as unknown as vscode.OutputChannel, asClineProvider(sidebarProvider)) + }) + + it("returns initial resource counters and lifecycle listener counts", () => { + sidebarProvider.getCurrentTaskStack.mockReturnValue(["root-task", "child-task"]) + + expect(api.getResourceDiagnostics()).toEqual({ + registeredTaskCount: 0, + currentTaskStackLength: 2, + listenerCounts: { + [RooCodeEventName.Message]: 0, + [RooCodeEventName.TaskCreated]: 0, + [RooCodeEventName.TaskStarted]: 0, + [RooCodeEventName.TaskCompleted]: 0, + [RooCodeEventName.TaskAborted]: 0, + [RooCodeEventName.TaskDelegationCompleted]: 0, + [RooCodeEventName.TaskDelegationResumed]: 0, + [RooCodeEventName.TaskModeSwitched]: 0, + }, + }) + }) + + it("reflects API lifecycle listener counts after on and off", () => { + const messageListener = () => undefined + const completedListener = () => undefined + + api.on(RooCodeEventName.Message, messageListener) + api.on(RooCodeEventName.TaskCompleted, completedListener) + + expect(api.getResourceDiagnostics().listenerCounts).toMatchObject({ + [RooCodeEventName.Message]: 1, + [RooCodeEventName.TaskCompleted]: 1, + }) + + api.off(RooCodeEventName.Message, messageListener) + + expect(api.getResourceDiagnostics().listenerCounts).toMatchObject({ + [RooCodeEventName.Message]: 0, + [RooCodeEventName.TaskCompleted]: 1, + }) + }) + + it("tracks registered task count across task creation, completion, and abortion", () => { + const completedTask = createTask("completed-task") + sidebarProvider.emit(RooCodeEventName.TaskCreated, completedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + completedTask.emit(RooCodeEventName.TaskCompleted, completedTask.taskId, {}, {}) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + + const abortedTask = createTask("aborted-task") + sidebarProvider.emit(RooCodeEventName.TaskCreated, abortedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + abortedTask.emit(RooCodeEventName.TaskAborted) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 2d0d5a6975..7832ce7ce7 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -10,6 +10,7 @@ import { type RooCodeAPI, type RooCodeSettings, type RooCodeEvents, + type RooCodeResourceDiagnostics, type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, @@ -30,11 +31,23 @@ import { openClineInNewTab } from "../activate/registerCommands" import { getCommands } from "../services/command/commands" import { getModels } from "../api/providers/fetchers/modelCache" +const RESOURCE_DIAGNOSTIC_EVENTS = [ + RooCodeEventName.Message, + RooCodeEventName.TaskCreated, + RooCodeEventName.TaskStarted, + RooCodeEventName.TaskCompleted, + RooCodeEventName.TaskAborted, + RooCodeEventName.TaskDelegationCompleted, + RooCodeEventName.TaskDelegationResumed, + RooCodeEventName.TaskModeSwitched, +] as const + export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider private readonly context: vscode.ExtensionContext private readonly ipc?: IpcServer + private readonly registeredTaskIds = new Set() private readonly log: (...args: unknown[]) => void private logfile?: string @@ -253,6 +266,18 @@ export class API extends EventEmitter implements RooCodeAPI { return this.sidebarProvider.getCurrentTaskStack() } + public getResourceDiagnostics(): RooCodeResourceDiagnostics { + const listenerCounts = Object.fromEntries( + RESOURCE_DIAGNOSTIC_EVENTS.map((eventName) => [eventName, this.listenerCount(eventName)]), + ) as RooCodeResourceDiagnostics["listenerCounts"] + + return { + registeredTaskCount: this.registeredTaskIds.size, + currentTaskStackLength: this.getCurrentTaskStack().length, + listenerCounts, + } + } + public async clearCurrentTask(_lastMessage?: string) { // Legacy finishSubTask removed; clear current by closing active task instance. await this.sidebarProvider.evictCurrentTask() @@ -329,6 +354,8 @@ export class API extends EventEmitter implements RooCodeAPI { private registerListeners(provider: ClineProvider) { provider.on(RooCodeEventName.TaskCreated, (task) => { + this.registeredTaskIds.add(task.taskId) + // Task Lifecycle task.on(RooCodeEventName.TaskStarted, async () => { @@ -340,6 +367,7 @@ export class API extends EventEmitter implements RooCodeAPI { this.emit(RooCodeEventName.TaskCompleted, task.taskId, tokenUsage, toolUsage, { isSubtask: !!task.parentTaskId, }) + this.registeredTaskIds.delete(task.taskId) await this.fileLog( `[${new Date().toISOString()}] taskCompleted -> ${task.taskId} | ${JSON.stringify(tokenUsage, null, 2)} | ${JSON.stringify(toolUsage, null, 2)}\n`, @@ -348,6 +376,7 @@ export class API extends EventEmitter implements RooCodeAPI { task.on(RooCodeEventName.TaskAborted, () => { this.emit(RooCodeEventName.TaskAborted, task.taskId) + this.registeredTaskIds.delete(task.taskId) }) task.on(RooCodeEventName.TaskFocused, () => { From e1bc569dbe4f0f3a22d47e1e7a821380627045c2 Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 01:17:51 +0800 Subject: [PATCH 4/9] test(vscode-e2e): assert orchestrator resources converge --- .../src/fixtures/resource-diagnostics.spec.ts | 74 ++++++++++++++++ .../src/fixtures/resource-diagnostics.ts | 84 +++++++++++++++++++ .../vscode-e2e/src/suite/orchestrator.test.ts | 26 +++++- 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts create mode 100644 apps/vscode-e2e/src/fixtures/resource-diagnostics.ts diff --git a/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts b/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts new file mode 100644 index 0000000000..9d9827a27d --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts @@ -0,0 +1,74 @@ +import type { RooCodeResourceDiagnosticEventName, RooCodeResourceDiagnostics } from "@roo-code/types" +import { describe, expect, it } from "vitest" + +import { assertResourceDiagnosticsConverged } from "./resource-diagnostics" +type DiagnosticsOverrides = Omit, "listenerCounts"> & { + listenerCounts?: Partial> +} + +const diagnostics = (overrides: DiagnosticsOverrides = {}): RooCodeResourceDiagnostics => ({ + registeredTaskCount: overrides.registeredTaskCount ?? 0, + currentTaskStackLength: overrides.currentTaskStackLength ?? 0, + listenerCounts: { + message: 0, + taskCreated: 0, + taskStarted: 0, + taskCompleted: 0, + taskAborted: 0, + taskDelegationCompleted: 0, + taskDelegationResumed: 0, + taskModeSwitched: 0, + ...overrides.listenerCounts, + }, +}) + +describe("resource diagnostics convergence helper", () => { + it("passes when final diagnostics match the baseline after cleanup", () => { + const baseline = diagnostics() + const final = diagnostics() + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).not.toThrow() + }) + + it("fails when registered task count increases", () => { + const baseline = diagnostics({ registeredTaskCount: 0 }) + const final = diagnostics({ registeredTaskCount: 1 }) + + expect(() => + assertResourceDiagnosticsConverged({ + baseline, + final, + observedChildTaskIds: ["child-1"], + }), + ).toThrow(/registeredTaskCount.*child-1/) + }) + + it("fails when listener counts increase", () => { + const baseline = diagnostics({ listenerCounts: { taskDelegationCompleted: 1 } }) + const final = diagnostics({ listenerCounts: { taskDelegationCompleted: 2 } }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow( + /listenerCounts\.taskDelegationCompleted/, + ) + }) + + it("fails when current task stack length has not converged", () => { + const baseline = diagnostics({ currentTaskStackLength: 0 }) + const final = diagnostics({ currentTaskStackLength: 1 }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow(/currentTaskStackLength/) + }) + + it("includes every leaking counter name in the error message", () => { + const baseline = diagnostics() + const final = diagnostics({ + registeredTaskCount: 2, + currentTaskStackLength: 1, + listenerCounts: { taskCompleted: 1, taskDelegationCompleted: 1 }, + }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow( + /registeredTaskCount[\s\S]*currentTaskStackLength[\s\S]*listenerCounts\.taskCompleted[\s\S]*listenerCounts\.taskDelegationCompleted/, + ) + }) +}) diff --git a/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts new file mode 100644 index 0000000000..d2a6133fa8 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts @@ -0,0 +1,84 @@ +import type { RooCodeResourceDiagnosticEventName, RooCodeResourceDiagnostics } from "@roo-code/types" + +export type ResourceDiagnosticsConvergenceOptions = { + baseline: RooCodeResourceDiagnostics + final: RooCodeResourceDiagnostics + observedChildTaskIds?: string[] +} + +type DiagnosticIssue = { + name: string + baseline: number + final: number +} + +const formatIssue = ({ name, baseline, final }: DiagnosticIssue) => `${name}: baseline=${baseline}, final=${final}` + +const listenerCountIssues = ( + baseline: RooCodeResourceDiagnostics, + final: RooCodeResourceDiagnostics, +): DiagnosticIssue[] => { + const listenerNames = new Set([ + ...(Object.keys(baseline.listenerCounts) as RooCodeResourceDiagnosticEventName[]), + ...(Object.keys(final.listenerCounts) as RooCodeResourceDiagnosticEventName[]), + ]) + + return [...listenerNames].sort().flatMap((listenerName) => { + const baselineCount = baseline.listenerCounts[listenerName] ?? 0 + const finalCount = final.listenerCounts[listenerName] ?? 0 + + return finalCount === baselineCount + ? [] + : [ + { + name: `listenerCounts.${listenerName}`, + baseline: baselineCount, + final: finalCount, + }, + ] + }) +} + +export const getResourceDiagnosticsConvergenceIssues = ({ + baseline, + final, +}: ResourceDiagnosticsConvergenceOptions): DiagnosticIssue[] => { + const issues: DiagnosticIssue[] = [] + + if (final.registeredTaskCount !== baseline.registeredTaskCount) { + issues.push({ + name: "registeredTaskCount", + baseline: baseline.registeredTaskCount, + final: final.registeredTaskCount, + }) + } + + if (final.currentTaskStackLength !== baseline.currentTaskStackLength && final.currentTaskStackLength !== 0) { + issues.push({ + name: "currentTaskStackLength", + baseline: baseline.currentTaskStackLength, + final: final.currentTaskStackLength, + }) + } + + issues.push(...listenerCountIssues(baseline, final)) + + return issues +} + +export const assertResourceDiagnosticsConverged = (options: ResourceDiagnosticsConvergenceOptions) => { + const issues = getResourceDiagnosticsConvergenceIssues(options) + + if (issues.length === 0) { + return + } + + const staleChildHint = + options.observedChildTaskIds && options.observedChildTaskIds.length > 0 + ? ` Observed child task ids: ${options.observedChildTaskIds.join(", ")}.` + : "" + + throw new Error( + `Resource diagnostics did not converge after orchestrator cleanup: ${issues.map(formatIssue).join("; ")}.${staleChildHint}`, + ) +} diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts index cf6b9a9fd2..afeeb3f785 100644 --- a/apps/vscode-e2e/src/suite/orchestrator.test.ts +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -4,6 +4,10 @@ import { RooCodeEventName, type ClineMessage } from "@roo-code/types" import { setDefaultSuiteTimeout } from "./test-utils" import { sleep, waitFor, waitUntilCompleted } from "./utils" +import { + assertResourceDiagnosticsConverged, + getResourceDiagnosticsConvergenceIssues, +} from "../fixtures/resource-diagnostics" import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, @@ -106,6 +110,7 @@ suite("Roo Code Orchestrator", function () { test("orchestrator parent repeats three rounds of delegated child fan-in without stack duplication", async () => { const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() const says: Record = {} const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] const parentStackSnapshots: string[][] = [] @@ -217,7 +222,26 @@ suite("Roo Code Orchestrator", function () { } await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after cleanup") - await sleep(100) + + const observedChildTaskIds = delegationCompletions.map(({ childId }) => childId) + await waitFor(() => { + const final = api.getResourceDiagnostics() + + return ( + getResourceDiagnosticsConvergenceIssues({ + baseline: baselineDiagnostics, + final, + observedChildTaskIds, + }).length === 0 + ) + }).catch(() => {}) + + const finalDiagnostics = api.getResourceDiagnostics() + assertResourceDiagnosticsConverged({ + baseline: baselineDiagnostics, + final: finalDiagnostics, + observedChildTaskIds, + }) } }) }) From 3182f4f725d84e55f41fb81797580189ab2bbcc1 Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 01:33:40 +0800 Subject: [PATCH 5/9] test(vscode-e2e): cover nested orchestrator delegation --- .../src/fixtures/orchestrator-plan.spec.ts | 99 ++++++++++++ .../src/fixtures/orchestrator-plan.ts | 88 ++++++++++- apps/vscode-e2e/src/fixtures/orchestrator.ts | 128 +++++++++++++++ .../vscode-e2e/src/suite/orchestrator.test.ts | 146 ++++++++++++++++++ 4 files changed, 457 insertions(+), 4 deletions(-) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts index 4b5d6e3e8c..74ed71fe17 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -4,13 +4,23 @@ import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_MARKER, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorNestedChildResumeExpectations, + buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorNestedChildResumeRequest, + shouldMatchOrchestratorNestedParentResumeRequest, shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" @@ -177,4 +187,93 @@ describe("orchestrator fan-out delegation plan", () => { expect(ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT).toContain(`Round ${round} ${role}: ${summary}`) } }) + + it("defines a nested parent/child orchestrator plan with A/B/C/D roles, modes, and summaries", () => { + expect(ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT).toContain(ORCHESTRATOR_NESTED_DELEGATION_MARKER) + expect(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP).toEqual( + expect.objectContaining({ + role: "child-orchestrator", + mode: "orchestrator", + summary: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + }), + ) + expect(ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS).toEqual([ + expect.objectContaining({ + role: "requirements", + mode: "ask", + summary: "Nested requirement summary: capture child orchestrator requirements.", + }), + expect.objectContaining({ + role: "implementation", + mode: "code", + summary: "Nested implementation summary: produce child orchestrator implementation notes.", + }), + ]) + }) + + it("matches nested child orchestrator resumes only after cumulative C/D result injection", () => { + const expectations = buildOrchestratorNestedChildResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: ["Nested requirement summary: capture child orchestrator requirements."], + nextMode: "code", + }, + { + stepIndex: 2, + requiredSummaries: ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.map(({ summary }) => summary), + nextMode: undefined, + }, + ]) + + const secondExpectation = expectations[1]! + const childResumeRequest = `${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker} completed.\\n\\nResult:\\n${secondExpectation.requiredSummaries.join(" completed.\\n\\nResult:\\n")}` + + expect( + shouldMatchOrchestratorNestedChildResumeRequest(childResumeRequest, secondExpectation.requiredSummaries), + ).toBe(true) + expect( + shouldMatchOrchestratorNestedChildResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker} completed.\\n\\nResult:\\nmissing ${secondExpectation.requiredSummaries[1]}`, + secondExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorNestedChildResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${secondExpectation.requiredSummaries[0]}`, + [secondExpectation.requiredSummaries[0]!], + ), + ).toBe(false) + }) + + it("matches top-level nested parent resumes only after the B nested result", () => { + const expectations = buildOrchestratorNestedParentResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: [ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT], + nextMode: undefined, + }, + ]) + + const parentResumeRequest = `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` + expect( + shouldMatchOrchestratorNestedParentResumeRequest(parentResumeRequest, [ + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ]), + ).toBe(true) + expect( + shouldMatchOrchestratorNestedParentResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary}`, + [ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT], + ), + ).toBe(false) + }) + + it("composes nested child and final parent results with the nested summaries", () => { + for (const { summary } of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { + expect(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT).toContain(summary) + } + expect(ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT).toContain(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + }) }) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts index 062e1edb35..210d47b5f1 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -1,4 +1,4 @@ -export type OrchestratorFanOutMode = "ask" | "architect" | "code" +export type OrchestratorFanOutMode = "ask" | "architect" | "code" | "orchestrator" export type OrchestratorFanOutChildStep = { readonly mode: OrchestratorFanOutMode @@ -10,12 +10,17 @@ export type OrchestratorFanOutChildStep = { } export type OrchestratorRepeatedDelegationRole = "requirements" | "design" | "implementation" +export type OrchestratorNestedDelegationRole = "child-orchestrator" | "requirements" | "implementation" export type OrchestratorRepeatedDelegationChildStep = OrchestratorFanOutChildStep & { readonly round: number readonly role: OrchestratorRepeatedDelegationRole } +export type OrchestratorNestedDelegationStep = OrchestratorFanOutChildStep & { + readonly role: OrchestratorNestedDelegationRole +} + export type OrchestratorResumeExpectation = { readonly stepIndex: number readonly requiredSummaries: string[] @@ -25,6 +30,7 @@ export type OrchestratorResumeExpectation = { export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT" export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:" export const ORCHESTRATOR_REPEATED_DELEGATION_MARKER = "ORCHESTRATOR_REPEATED_DELEGATION_STRESS" +export const ORCHESTRATOR_NESTED_DELEGATION_MARKER = "ORCHESTRATOR_NESTED_DELEGATION" export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [ { @@ -115,6 +121,43 @@ export const ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_RE export const ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT = `Orchestrator repeated delegation complete:\n${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, summary }) => `- Round ${round} ${role}: ${summary}`).join("\\n")}` +export const ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS: readonly OrchestratorNestedDelegationStep[] = [ + { + role: "requirements", + mode: "ask", + marker: "ORCHESTRATOR_NESTED_REQUIREMENTS_GRANDCHILD", + prompt: 'ORCHESTRATOR_NESTED_REQUIREMENTS_GRANDCHILD: Complete immediately with the exact result "Nested requirement summary: capture child orchestrator requirements."', + summary: "Nested requirement summary: capture child orchestrator requirements.", + newTaskToolCallId: "call_orchestrator_nested_child_new_task_001", + completionToolCallId: "call_orchestrator_nested_requirements_completion_001", + }, + { + role: "implementation", + mode: "code", + marker: "ORCHESTRATOR_NESTED_IMPLEMENTATION_GRANDCHILD", + prompt: 'ORCHESTRATOR_NESTED_IMPLEMENTATION_GRANDCHILD: Complete immediately with the exact result "Nested implementation summary: produce child orchestrator implementation notes."', + summary: "Nested implementation summary: produce child orchestrator implementation notes.", + newTaskToolCallId: "call_orchestrator_nested_child_new_task_002", + completionToolCallId: "call_orchestrator_nested_implementation_completion_001", + }, +] + +export const ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT = `Nested child orchestrator complete:\n- C: ${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary}\n- D: ${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.summary}` + +export const ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP: OrchestratorNestedDelegationStep = { + role: "child-orchestrator", + mode: "orchestrator", + marker: "ORCHESTRATOR_NESTED_CHILD_ORCHESTRATOR", + prompt: `ORCHESTRATOR_NESTED_CHILD_ORCHESTRATOR: Delegate exactly two grandchildren in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.prompt}". After both grandchildren return, complete with the nested summary containing both grandchild summaries.`, + summary: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + newTaskToolCallId: "call_orchestrator_nested_parent_new_task_001", + completionToolCallId: "call_orchestrator_nested_child_orchestrator_completion_001", +} + +export const ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_NESTED_DELEGATION_MARKER}: Delegate exactly one child orchestrator B in orchestrator mode with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.prompt}". After B returns, complete with a final top-level summary containing B's nested result.` + +export const ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT = `Nested top-level orchestrator complete:\n- B: ${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` + function buildResumeExpectations(steps: readonly OrchestratorFanOutChildStep[]): OrchestratorResumeExpectation[] { return steps.map((_step, index) => ({ stepIndex: index + 1, @@ -131,13 +174,28 @@ export function buildOrchestratorRepeatedResumeExpectations(): OrchestratorResum return buildResumeExpectations(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) } +export function buildOrchestratorNestedChildResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) +} + +export function buildOrchestratorNestedParentResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations([ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP]) +} + export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean { return ( rawRequest.includes(childMarker) && !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && - !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) + !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker) ) } +function requestContainsResultSummary(rawRequest: string, summary: string): boolean { + const jsonEscapedSummary = summary.replaceAll("\n", "\\n") + + return rawRequest.includes(`Result:\\n${summary}`) || rawRequest.includes(`Result:\\n${jsonEscapedSummary}`) +} export function shouldMatchOrchestratorResumeRequest( rawRequest: string, @@ -146,7 +204,7 @@ export function shouldMatchOrchestratorResumeRequest( return ( rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && - requiredSummaries.every((summary) => rawRequest.includes(`Result:\\n${summary}`)) + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) ) } @@ -157,6 +215,28 @@ export function shouldMatchOrchestratorRepeatedResumeRequest( return ( rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && - requiredSummaries.every((summary) => rawRequest.includes(`Result:\\n${summary}`)) + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorNestedChildResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorNestedParentResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) ) } diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts index ba3694c9f8..e47c137615 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -7,13 +7,23 @@ import { ORCHESTRATOR_FAN_OUT_MARKER, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_MARKER, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorNestedChildResumeExpectations, + buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorNestedChildResumeRequest, + shouldMatchOrchestratorNestedParentResumeRequest, shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" @@ -24,13 +34,23 @@ export { ORCHESTRATOR_FAN_OUT_MARKER, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_MARKER, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorNestedChildResumeExpectations, + buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorNestedChildResumeRequest, + shouldMatchOrchestratorNestedParentResumeRequest, shouldMatchOrchestratorRepeatedResumeRequest, shouldMatchOrchestratorResumeRequest, } @@ -40,6 +60,7 @@ const requestText = (req: ChatCompletionRequest) => JSON.stringify(req) export function addOrchestratorFixtures(mock: InstanceType) { const firstFanOutStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]! const firstRepeatedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]! + const firstNestedGrandchildStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]! mock.addFixture({ match: { @@ -175,4 +196,111 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) } + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_NESTED_DELEGATION_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.mode, + message: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.prompt, + }), + id: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.newTaskToolCallId, + }, + ], + }, + }) + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstNestedGrandchildStep.mode, + message: firstNestedGrandchildStep.prompt, + }), + id: firstNestedGrandchildStep.newTaskToolCallId, + }, + ], + }, + }) + + for (const step of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorChildRequest(requestText(req), step.marker), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: step.summary }), + id: step.completionToolCallId, + }, + ], + }, + }) + } + + for (const expectation of [...buildOrchestratorNestedChildResumeExpectations()].reverse()) { + const nextStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[expectation.stepIndex] + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorNestedChildResumeRequest(requestText(req), expectation.requiredSummaries), + }, + response: { + toolCalls: nextStep + ? [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: nextStep.mode, + message: nextStep.prompt, + }), + id: nextStep.newTaskToolCallId, + }, + ] + : [ + { + name: "attempt_completion", + arguments: JSON.stringify({ + result: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + }), + id: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.completionToolCallId, + }, + ], + }, + }) + } + + for (const expectation of [...buildOrchestratorNestedParentResumeExpectations()].reverse()) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorNestedParentResumeRequest(requestText(req), expectation.requiredSummaries), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT }), + id: "call_orchestrator_nested_parent_completion_002", + }, + ], + }, + }) + } } diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts index afeeb3f785..15c83dd75c 100644 --- a/apps/vscode-e2e/src/suite/orchestrator.test.ts +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -12,6 +12,11 @@ import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, @@ -244,4 +249,145 @@ suite("Roo Code Orchestrator", function () { }) } }) + + test("orchestrator parent fans in through a nested child orchestrator", async () => { + const api = globalThis.api + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === 3) + + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + [ + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.summary, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ], + "Nested fan-in events should complete C/D before B completes to A", + ) + + const childOrchestratorCompletion = delegationCompletions.find( + ({ parentId, summary }) => + parentId === parentTaskId && summary === ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ) + assert.ok(childOrchestratorCompletion, "A should receive B's nested orchestrator completion") + + const childOrchestratorId = childOrchestratorCompletion.childId + const grandchildCompletions = delegationCompletions.filter( + ({ parentId }) => parentId === childOrchestratorId, + ) + const grandchildIds = grandchildCompletions.map(({ childId }) => childId) + assert.deepStrictEqual( + grandchildCompletions.map(({ summary }) => summary), + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.map(({ summary }) => summary), + "C/D completions should resume only B before B resumes A", + ) + assert.ok( + delegationCompletions + .filter(({ summary }) => + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.some((step) => step.summary === summary), + ) + .every(({ parentId }) => parentId === childOrchestratorId), + "C/D completion events should not directly resume A", + ) + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Nested parent A history item should exist") + assert.strictEqual(parent.status, "completed", "Nested parent A should complete after B fan-in") + assert.deepStrictEqual(parent.childIds, [childOrchestratorId], "A childIds should contain only B") + + const childOrchestrator = await api.getTaskHistoryItem(childOrchestratorId) + assert.ok(childOrchestrator, "Nested child orchestrator B history item should exist") + assert.strictEqual(childOrchestrator.parentTaskId, parentTaskId, "B parentTaskId should point to A") + assert.strictEqual(childOrchestrator.mode, ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.mode) + assert.strictEqual(childOrchestrator.status, "completed", "B should complete after C/D fan-in") + assert.deepStrictEqual(childOrchestrator.childIds, grandchildIds, "B childIds should contain C/D") + assert.strictEqual( + childOrchestrator.completionResultSummary, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + "B completion summary should include nested C/D result", + ) + + for (const [index, grandchildId] of grandchildIds.entries()) { + const expectedStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[index]! + const grandchild = await api.getTaskHistoryItem(grandchildId) + assert.ok(grandchild, `Nested grandchild ${index + 1} history item should exist`) + assert.strictEqual( + grandchild.parentTaskId, + childOrchestratorId, + `Grandchild ${index + 1} should point to B`, + ) + assert.strictEqual( + grandchild.mode, + expectedStep.mode, + `Grandchild ${index + 1} mode should match the nested plan`, + ) + assert.strictEqual(grandchild.status, "completed", `Grandchild ${index + 1} should be completed`) + assert.strictEqual( + grandchild.completionResultSummary, + expectedStep.summary, + `Grandchild ${index + 1} summary should be persisted`, + ) + } + + const childCompletionText = says[childOrchestratorId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text) => text === ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + assert.strictEqual(childCompletionText, ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + for (const { summary } of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { + assert.ok(childCompletionText.includes(summary), `B completion should include nested ${summary}`) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text) => text === ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT) + assert.strictEqual(parentCompletionText, ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT) + assert.ok( + parentCompletionText.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT), + "Final top-level completion should include B's nested summary", + ) + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + while (api.getCurrentTaskStack().length > 0) { + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after nested cleanup") + await sleep(100) + } + }) }) From 4c5c41d03150b7d1779d5bae0c19373815950f9b Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 01:45:05 +0800 Subject: [PATCH 6/9] test(vscode-e2e): cover orchestrator cancellation recovery --- .../src/fixtures/orchestrator-plan.spec.ts | 89 ++++++++++ .../src/fixtures/orchestrator-plan.ts | 75 ++++++++ apps/vscode-e2e/src/fixtures/orchestrator.ts | 93 ++++++++++ .../vscode-e2e/src/suite/orchestrator.test.ts | 162 ++++++++++++++++++ 4 files changed, 419 insertions(+) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts index 74ed71fe17..11b17d91ba 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -3,6 +3,11 @@ import { describe, expect, it } from "vitest" import { ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, @@ -14,10 +19,14 @@ import { ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorCancellationRecoveryResumeExpectations, buildOrchestratorNestedChildResumeExpectations, buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, + shouldMatchOrchestratorCancellationChildCompletionRequest, + shouldMatchOrchestratorCancellationChildRequest, + shouldMatchOrchestratorCancellationRecoveryResumeRequest, shouldMatchOrchestratorChildRequest, shouldMatchOrchestratorNestedChildResumeRequest, shouldMatchOrchestratorNestedParentResumeRequest, @@ -276,4 +285,84 @@ describe("orchestrator fan-out delegation plan", () => { } expect(ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT).toContain(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) }) + + it("defines a cancellation recovery parent/child plan with a pending ask child", () => { + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT).toContain(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP).toEqual( + expect.objectContaining({ + mode: "ask", + role: "cancellation-child", + summary: "Cancellation recovery child summary: resumed after interruption.", + }), + ) + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain("Ask the user exactly") + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain( + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ) + }) + + it("matches cancellation child requests without colliding with other orchestrator scenarios", () => { + expect( + shouldMatchOrchestratorCancellationChildRequest( + `child ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`, + ), + ).toBe(true) + expect(shouldMatchOrchestratorCancellationChildRequest(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT)).toBe( + false, + ) + expect( + shouldMatchOrchestratorCancellationChildRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} embeds ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`, + ), + ).toBe(false) + }) + + it("matches cancellation child completion only after the recovery answer", () => { + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} call_orchestrator_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, + ), + ).toBe(true) + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} without the answer`, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} call_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, + ), + ).toBe(false) + }) + + it("matches cancellation recovery parent resumes only after child result injection", () => { + const expectations = buildOrchestratorCancellationRecoveryResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: [ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary], + nextMode: undefined, + }, + ]) + + const parentResumeRequest = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}` + expect( + shouldMatchOrchestratorCancellationRecoveryResumeRequest(parentResumeRequest, [ + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + ]), + ).toBe(true) + expect( + shouldMatchOrchestratorCancellationRecoveryResumeRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\nmissing`, + [ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary], + ), + ).toBe(false) + }) + + it("composes cancellation recovery final result with an explicit cancellation marker", () => { + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain("cancellation recovery") + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain( + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + ) + }) }) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts index 210d47b5f1..21650dafa8 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -11,6 +11,7 @@ export type OrchestratorFanOutChildStep = { export type OrchestratorRepeatedDelegationRole = "requirements" | "design" | "implementation" export type OrchestratorNestedDelegationRole = "child-orchestrator" | "requirements" | "implementation" +export type OrchestratorCancellationRecoveryRole = "cancellation-child" export type OrchestratorRepeatedDelegationChildStep = OrchestratorFanOutChildStep & { readonly round: number @@ -21,6 +22,11 @@ export type OrchestratorNestedDelegationStep = OrchestratorFanOutChildStep & { readonly role: OrchestratorNestedDelegationRole } +export type OrchestratorCancellationRecoveryStep = OrchestratorFanOutChildStep & { + readonly role: OrchestratorCancellationRecoveryRole + readonly followupAnswer: string +} + export type OrchestratorResumeExpectation = { readonly stepIndex: number readonly requiredSummaries: string[] @@ -31,6 +37,7 @@ export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT" export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:" export const ORCHESTRATOR_REPEATED_DELEGATION_MARKER = "ORCHESTRATOR_REPEATED_DELEGATION_STRESS" export const ORCHESTRATOR_NESTED_DELEGATION_MARKER = "ORCHESTRATOR_NESTED_DELEGATION" +export const ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER = "ORCHESTRATOR_CANCELLATION_RECOVERY" export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [ { @@ -158,6 +165,23 @@ export const ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_NEST export const ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT = `Nested top-level orchestrator complete:\n- B: ${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER = "resume recovered cancellation child" + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP: OrchestratorCancellationRecoveryStep = { + role: "cancellation-child", + mode: "ask", + marker: "ORCH_CANCEL_RECOVERY_CHILD", + prompt: `ORCH_CANCEL_RECOVERY_CHILD: Ask the user exactly this follow-up question: Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}" to recover the cancelled child. After the user answers, complete with the exact result "Cancellation recovery child summary: resumed after interruption."`, + summary: "Cancellation recovery child summary: resumed after interruption.", + followupAnswer: ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + newTaskToolCallId: "call_orchestrator_cancellation_parent_new_task_001", + completionToolCallId: "call_orchestrator_cancellation_child_completion_002", +} + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B with this exact message: "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt}". Wait for B to return; do not complete before B returns. After B returns, complete with a final cancellation recovery summary containing B's result.` + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT = `Orchestrator cancellation recovery complete:\n- B: ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}` + function buildResumeExpectations(steps: readonly OrchestratorFanOutChildStep[]): OrchestratorResumeExpectation[] { return steps.map((_step, index) => ({ stepIndex: index + 1, @@ -182,15 +206,55 @@ export function buildOrchestratorNestedParentResumeExpectations(): OrchestratorR return buildResumeExpectations([ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP]) } +export function buildOrchestratorCancellationRecoveryResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations([ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP]) +} + export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean { return ( rawRequest.includes(childMarker) && !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) && !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker) ) } + +function shouldMatchOrchestratorCancellationChildBase(rawRequest: string): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker) && + !rawRequest.includes(`${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B`) && + !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) + ) +} + +export function shouldMatchOrchestratorCancellationChildRequest(rawRequest: string): boolean { + const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace( + "completion_002", + "followup_001", + ) + + return shouldMatchOrchestratorCancellationChildBase(rawRequest) && !rawRequest.includes(followupToolCallId) +} + +export function shouldMatchOrchestratorCancellationChildCompletionRequest(rawRequest: string): boolean { + const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace( + "completion_002", + "followup_001", + ) + + return ( + shouldMatchOrchestratorCancellationChildBase(rawRequest) && + ((rawRequest.includes(followupToolCallId) && + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER)) || + rawRequest.includes( + `\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}\\n`, + )) + ) +} function requestContainsResultSummary(rawRequest: string, summary: string): boolean { const jsonEscapedSummary = summary.replaceAll("\n", "\\n") @@ -240,3 +304,14 @@ export function shouldMatchOrchestratorNestedParentResumeRequest( requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) ) } + +export function shouldMatchOrchestratorCancellationRecoveryResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts index e47c137615..b30ffac752 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -2,6 +2,11 @@ import { LLMock } from "@copilotkit/aimock" import type { ChatCompletionRequest } from "@copilotkit/aimock" import { + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_MARKER, @@ -17,10 +22,14 @@ import { ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorCancellationRecoveryResumeExpectations, buildOrchestratorNestedChildResumeExpectations, buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, + shouldMatchOrchestratorCancellationChildCompletionRequest, + shouldMatchOrchestratorCancellationChildRequest, + shouldMatchOrchestratorCancellationRecoveryResumeRequest, shouldMatchOrchestratorChildRequest, shouldMatchOrchestratorNestedChildResumeRequest, shouldMatchOrchestratorNestedParentResumeRequest, @@ -29,6 +38,11 @@ import { } from "./orchestrator-plan" export { + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_MARKER, @@ -44,10 +58,14 @@ export { ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorCancellationRecoveryResumeExpectations, buildOrchestratorNestedChildResumeExpectations, buildOrchestratorNestedParentResumeExpectations, buildOrchestratorRepeatedResumeExpectations, buildOrchestratorResumeExpectations, + shouldMatchOrchestratorCancellationChildCompletionRequest, + shouldMatchOrchestratorCancellationChildRequest, + shouldMatchOrchestratorCancellationRecoveryResumeRequest, shouldMatchOrchestratorChildRequest, shouldMatchOrchestratorNestedChildResumeRequest, shouldMatchOrchestratorNestedParentResumeRequest, @@ -303,4 +321,79 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) } + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.mode, + message: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt, + }), + id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.newTaskToolCallId, + }, + ], + }, + }) + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorCancellationChildRequest(requestText(req)), + }, + response: { + toolCalls: [ + { + name: "ask_followup_question", + arguments: JSON.stringify({ + question: `Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer}" to recover the cancelled child.`, + follow_up: [{ text: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer }], + }), + id: "call_orchestrator_cancellation_child_followup_001", + }, + ], + }, + }) + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorCancellationChildCompletionRequest(requestText(req)), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary }), + id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId, + }, + ], + }, + }) + + for (const expectation of [...buildOrchestratorCancellationRecoveryResumeExpectations()].reverse()) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorCancellationRecoveryResumeRequest( + requestText(req), + expectation.requiredSummaries, + ), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT }), + id: "call_orchestrator_cancellation_parent_completion_002", + }, + ], + }, + }) + } } diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts index 15c83dd75c..de8af5c7fc 100644 --- a/apps/vscode-e2e/src/suite/orchestrator.test.ts +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -9,6 +9,10 @@ import { getResourceDiagnosticsConvergenceIssues, } from "../fixtures/resource-diagnostics" import { + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, @@ -390,4 +394,162 @@ suite("Roo Code Orchestrator", function () { await sleep(100) } }) + + test("orchestrator child cancellation stays delegated until explicit recovery", async () => { + const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() + const asks: Record = {} + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "ask") { + asks[taskId] = asks[taskId] || [] + asks[taskId].push(message) + } + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, + }) + + let childTaskId: string | undefined + await waitFor(() => { + const stack = api.getCurrentTaskStack() + const current = stack[stack.length - 1] + if (current && current !== parentTaskId) { + childTaskId = current + return true + } + return false + }) + + await waitFor(() => asks[childTaskId!]?.some(({ ask }) => ask === "followup") ?? false) + await waitFor(async () => (await api.getTaskApiConversationHistoryLength(childTaskId!)) > 0) + + const delegatedChild = await api.getTaskHistoryItem(childTaskId!) + assert.ok(delegatedChild, "Cancellation child history item should exist after delegation") + assert.strictEqual(delegatedChild.parentTaskId, parentTaskId, "B parentTaskId should point to A") + assert.strictEqual(delegatedChild.mode, ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.mode) + + await api.cancelCurrentTask() + + await waitFor(() => api.getCurrentTaskStack().at(-1) === childTaskId) + await waitFor( + () => asks[childTaskId!]?.some(({ type, ask }) => type === "ask" && ask === "resume_task") ?? false, + ) + + const interruptedChild = await api.getTaskHistoryItem(childTaskId!) + assert.strictEqual(interruptedChild?.status, "interrupted", "B should be interrupted after cancellation") + assert.strictEqual( + interruptedChild?.parentTaskId, + parentTaskId, + "Interrupted B should retain its parent link to A", + ) + + const delegatedParent = await api.getTaskHistoryItem(parentTaskId) + assert.strictEqual(delegatedParent?.status, "delegated", "A should stay delegated after B cancellation") + assert.strictEqual(delegatedParent?.awaitingChildId, childTaskId, "A should keep awaiting interrupted B") + assert.strictEqual(delegationCompletions.length, 0, "B cancellation should not emit a completion to A") + assert.strictEqual( + says[parentTaskId]?.find(({ say }) => say === "completion_result"), + undefined, + "A must not receive premature final completion after B cancellation", + ) + + const completedParentTaskId = await waitUntilCompleted({ + api, + start: async () => { + await api.sendMessage(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER) + return parentTaskId + }, + }) + + assert.strictEqual( + completedParentTaskId, + parentTaskId, + "Explicitly recovered B should report back and complete A", + ) + assert.deepStrictEqual( + delegationCompletions, + [ + { + parentId: parentTaskId, + childId: childTaskId, + summary: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + }, + ], + "Recovered B should emit exactly one completion to A", + ) + + const recoveredChild = await api.getTaskHistoryItem(childTaskId!) + assert.strictEqual(recoveredChild?.status, "completed", "B should complete after explicit recovery") + assert.strictEqual( + recoveredChild?.completionResultSummary, + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + "B recovery summary should be persisted", + ) + + const completedParent = await api.getTaskHistoryItem(parentTaskId) + assert.strictEqual(completedParent?.status, "completed", "A should complete after explicit recovery") + assert.deepStrictEqual(completedParent?.childIds, [childTaskId], "A childIds should contain only B") + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => text === ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT) + assert.strictEqual(parentCompletionText, ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT) + assert.ok( + parentCompletionText.includes("cancellation recovery"), + "A final summary should explicitly identify cancellation recovery", + ) + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + while (api.getCurrentTaskStack().length > 0) { + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after recovery cleanup") + + const observedChildTaskIds = delegationCompletions.map(({ childId }) => childId) + await waitFor(() => { + const final = api.getResourceDiagnostics() + + return ( + getResourceDiagnosticsConvergenceIssues({ + baseline: baselineDiagnostics, + final, + observedChildTaskIds, + }).length === 0 + ) + }).catch(() => {}) + + const finalDiagnostics = api.getResourceDiagnostics() + assertResourceDiagnosticsConverged({ + baseline: baselineDiagnostics, + final: finalDiagnostics, + observedChildTaskIds, + }) + } + }) }) From a9f7e0bd19e3eefb65e7577824232e14fbffa11c Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 02:14:43 +0800 Subject: [PATCH 7/9] ci(vscode-e2e): run fixture unit tests --- .github/workflows/e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2d1819b4e1..9933dbde84 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -144,6 +144,10 @@ jobs: apps/vscode-e2e/.vscode-test/ key: vscode-test-${{ runner.os }}-${{ steps.vscode-ver.outputs.version }}-v1 + - name: Run vscode-e2e unit tests + if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' + run: pnpm --filter @roo-code/vscode-e2e test:unit + - name: Run mocked E2E tests id: run-e2e # merge_group and workflow_dispatch always run; cache skip is pull_request only From 62d3226c9f11b2e4db9c28dbf3d12aa837897f5a Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 02:35:28 +0800 Subject: [PATCH 8/9] fix(e2e): address orchestrator review feedback --- .github/workflows/e2e.yml | 8 +- .../src/fixtures/orchestrator-plan.spec.ts | 5 +- .../src/fixtures/orchestrator-plan.ts | 19 +- apps/vscode-e2e/src/fixtures/orchestrator.ts | 299 ++++++------------ .../src/fixtures/resource-diagnostics.ts | 1 + .../vscode-e2e/src/suite/orchestrator.test.ts | 154 +++++---- packages/types/src/api.ts | 6 +- .../api-resource-diagnostics.spec.ts | 14 + src/extension/api.ts | 15 +- 9 files changed, 230 insertions(+), 291 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9933dbde84..7c3545a9b1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -34,6 +34,10 @@ jobs: - name: Setup Node.js and pnpm if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' uses: ./.github/actions/setup-node-pnpm + - name: Run vscode-e2e unit tests + if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' + run: pnpm --filter @roo-code/vscode-e2e test:unit + - name: Install xvfb if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' run: sudo apt-get install -y xvfb @@ -144,10 +148,6 @@ jobs: apps/vscode-e2e/.vscode-test/ key: vscode-test-${{ runner.os }}-${{ steps.vscode-ver.outputs.version }}-v1 - - name: Run vscode-e2e unit tests - if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' - run: pnpm --filter @roo-code/vscode-e2e test:unit - - name: Run mocked E2E tests id: run-e2e # merge_group and workflow_dispatch always run; cache skip is pull_request only diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts index 11b17d91ba..71c8320e9d 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -6,6 +6,7 @@ import { ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, @@ -34,7 +35,7 @@ import { shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" -describe("orchestrator fan-out delegation plan", () => { +describe("orchestrator delegation plans", () => { it("defines the first commit single-round ask/architect/code child sequence", () => { expect(ORCHESTRATOR_FAN_OUT_PARENT_PROMPT).toContain("ORCHESTRATOR_SINGLE_ROUND_FAN_OUT") expect(ORCHESTRATOR_FAN_OUT_CHILD_STEPS).toEqual([ @@ -320,7 +321,7 @@ describe("orchestrator fan-out delegation plan", () => { it("matches cancellation child completion only after the recovery answer", () => { expect( shouldMatchOrchestratorCancellationChildCompletionRequest( - `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} call_orchestrator_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, + `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID} ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, ), ).toBe(true) expect( diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts index 21650dafa8..c0ffba72f7 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -126,7 +126,7 @@ export const ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS: readonly Orchestrator export const ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER}: Run exactly three rounds. In each round, delegate exactly three children in order: ask-mode requirements, architect-mode design, then code-mode implementation. Use these exact child messages in order: ${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ prompt }) => `"${prompt}"`).join("; ")}. After each child returns, resume the parent before creating the next child. After all nine children return, complete with a final summary containing every round and child summary.` -export const ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT = `Orchestrator repeated delegation complete:\n${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, summary }) => `- Round ${round} ${role}: ${summary}`).join("\\n")}` +export const ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT = `Orchestrator repeated delegation complete:\n${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, summary }) => `- Round ${round} ${role}: ${summary}`).join("\n")}` export const ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS: readonly OrchestratorNestedDelegationStep[] = [ { @@ -166,6 +166,8 @@ export const ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_NEST export const ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT = `Nested top-level orchestrator complete:\n- B: ${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER = "resume recovered cancellation child" +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID = + "call_orchestrator_cancellation_child_followup_001" export const ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP: OrchestratorCancellationRecoveryStep = { role: "cancellation-child", @@ -232,23 +234,16 @@ function shouldMatchOrchestratorCancellationChildBase(rawRequest: string): boole } export function shouldMatchOrchestratorCancellationChildRequest(rawRequest: string): boolean { - const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace( - "completion_002", - "followup_001", + return ( + shouldMatchOrchestratorCancellationChildBase(rawRequest) && + !rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID) ) - - return shouldMatchOrchestratorCancellationChildBase(rawRequest) && !rawRequest.includes(followupToolCallId) } export function shouldMatchOrchestratorCancellationChildCompletionRequest(rawRequest: string): boolean { - const followupToolCallId = ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId.replace( - "completion_002", - "followup_001", - ) - return ( shouldMatchOrchestratorCancellationChildBase(rawRequest) && - ((rawRequest.includes(followupToolCallId) && + ((rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID) && rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER)) || rawRequest.includes( `\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}\\n`, diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts index b30ffac752..c7d3cb98d4 100644 --- a/apps/vscode-e2e/src/fixtures/orchestrator.ts +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -1,27 +1,24 @@ import { LLMock } from "@copilotkit/aimock" import type { ChatCompletionRequest } from "@copilotkit/aimock" +export * from "./orchestrator-plan" + import { ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, - ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, - ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, ORCHESTRATOR_FAN_OUT_CHILD_STEPS, ORCHESTRATOR_FAN_OUT_FINAL_RESULT, ORCHESTRATOR_FAN_OUT_MARKER, - ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, - ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, ORCHESTRATOR_NESTED_DELEGATION_MARKER, - ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, ORCHESTRATOR_REPEATED_DELEGATION_MARKER, - ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, buildOrchestratorCancellationRecoveryResumeExpectations, buildOrchestratorNestedChildResumeExpectations, buildOrchestratorNestedParentResumeExpectations, @@ -37,69 +34,13 @@ import { shouldMatchOrchestratorResumeRequest, } from "./orchestrator-plan" -export { - ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, - ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, - ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, - ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, - ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, - ORCHESTRATOR_FAN_OUT_CHILD_STEPS, - ORCHESTRATOR_FAN_OUT_FINAL_RESULT, - ORCHESTRATOR_FAN_OUT_MARKER, - ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, - ORCHESTRATOR_FAN_OUT_RESULT_INJECTION, - ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, - ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, - ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, - ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, - ORCHESTRATOR_NESTED_DELEGATION_MARKER, - ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, - ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, - ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, - ORCHESTRATOR_REPEATED_DELEGATION_MARKER, - ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, - buildOrchestratorCancellationRecoveryResumeExpectations, - buildOrchestratorNestedChildResumeExpectations, - buildOrchestratorNestedParentResumeExpectations, - buildOrchestratorRepeatedResumeExpectations, - buildOrchestratorResumeExpectations, - shouldMatchOrchestratorCancellationChildCompletionRequest, - shouldMatchOrchestratorCancellationChildRequest, - shouldMatchOrchestratorCancellationRecoveryResumeRequest, - shouldMatchOrchestratorChildRequest, - shouldMatchOrchestratorNestedChildResumeRequest, - shouldMatchOrchestratorNestedParentResumeRequest, - shouldMatchOrchestratorRepeatedResumeRequest, - shouldMatchOrchestratorResumeRequest, -} - const requestText = (req: ChatCompletionRequest) => JSON.stringify(req) -export function addOrchestratorFixtures(mock: InstanceType) { - const firstFanOutStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]! - const firstRepeatedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]! - const firstNestedGrandchildStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]! - - mock.addFixture({ - match: { - userMessage: new RegExp(ORCHESTRATOR_FAN_OUT_MARKER), - sequenceIndex: 0, - }, - response: { - toolCalls: [ - { - name: "new_task", - arguments: JSON.stringify({ - mode: firstFanOutStep.mode, - message: firstFanOutStep.prompt, - }), - id: firstFanOutStep.newTaskToolCallId, - }, - ], - }, - }) - - for (const step of ORCHESTRATOR_FAN_OUT_CHILD_STEPS) { +function addChildCompletionFixtures( + mock: InstanceType, + steps: readonly { marker: string; summary: string; completionToolCallId: string }[], +) { + for (const step of steps) { mock.addFixture({ match: { predicate: (req: ChatCompletionRequest) => @@ -116,14 +57,26 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) } +} - for (const expectation of [...buildOrchestratorResumeExpectations()].reverse()) { - const nextStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[expectation.stepIndex] +function addResumeFixtures( + mock: InstanceType, + config: { + expectations: readonly { stepIndex: number; requiredSummaries: readonly string[] }[] + steps: readonly { mode: string; prompt: string; newTaskToolCallId: string }[] + matches: (rawRequest: string, requiredSummaries: readonly string[]) => boolean + finalResult: string + finalToolCallId: string + }, +) { + // Register most-cumulative expectations first so less-specific predicates cannot shadow later rounds. + for (const expectation of [...config.expectations].reverse()) { + const nextStep = config.steps[expectation.stepIndex] mock.addFixture({ match: { predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorResumeRequest(requestText(req), expectation.requiredSummaries), + config.matches(requestText(req), expectation.requiredSummaries), }, response: { toolCalls: nextStep @@ -140,13 +93,47 @@ export function addOrchestratorFixtures(mock: InstanceType) { : [ { name: "attempt_completion", - arguments: JSON.stringify({ result: ORCHESTRATOR_FAN_OUT_FINAL_RESULT }), - id: "call_orchestrator_fan_out_parent_completion_004", + arguments: JSON.stringify({ result: config.finalResult }), + id: config.finalToolCallId, }, ], }, }) } +} + +export function addOrchestratorFixtures(mock: InstanceType) { + const firstFanOutStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]! + const firstRepeatedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]! + const firstNestedGrandchildStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]! + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_FAN_OUT_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstFanOutStep.mode, + message: firstFanOutStep.prompt, + }), + id: firstFanOutStep.newTaskToolCallId, + }, + ], + }, + }) + + addChildCompletionFixtures(mock, ORCHESTRATOR_FAN_OUT_CHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorResumeExpectations(), + steps: ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + matches: shouldMatchOrchestratorResumeRequest, + finalResult: ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + finalToolCallId: "call_orchestrator_fan_out_parent_completion_004", + }) mock.addFixture({ match: { userMessage: new RegExp(ORCHESTRATOR_REPEATED_DELEGATION_MARKER), @@ -166,54 +153,14 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) - for (const step of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorChildRequest(requestText(req), step.marker), - }, - response: { - toolCalls: [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: step.summary }), - id: step.completionToolCallId, - }, - ], - }, - }) - } - - for (const expectation of [...buildOrchestratorRepeatedResumeExpectations()].reverse()) { - const nextStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[expectation.stepIndex] - - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorRepeatedResumeRequest(requestText(req), expectation.requiredSummaries), - }, - response: { - toolCalls: nextStep - ? [ - { - name: "new_task", - arguments: JSON.stringify({ - mode: nextStep.mode, - message: nextStep.prompt, - }), - id: nextStep.newTaskToolCallId, - }, - ] - : [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT }), - id: "call_orchestrator_repeated_parent_completion_010", - }, - ], - }, - }) - } + addChildCompletionFixtures(mock, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorRepeatedResumeExpectations(), + steps: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + matches: shouldMatchOrchestratorRepeatedResumeRequest, + finalResult: ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + finalToolCallId: "call_orchestrator_repeated_parent_completion_010", + }) mock.addFixture({ match: { @@ -253,74 +200,21 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) - for (const step of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorChildRequest(requestText(req), step.marker), - }, - response: { - toolCalls: [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: step.summary }), - id: step.completionToolCallId, - }, - ], - }, - }) - } - - for (const expectation of [...buildOrchestratorNestedChildResumeExpectations()].reverse()) { - const nextStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[expectation.stepIndex] - - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorNestedChildResumeRequest(requestText(req), expectation.requiredSummaries), - }, - response: { - toolCalls: nextStep - ? [ - { - name: "new_task", - arguments: JSON.stringify({ - mode: nextStep.mode, - message: nextStep.prompt, - }), - id: nextStep.newTaskToolCallId, - }, - ] - : [ - { - name: "attempt_completion", - arguments: JSON.stringify({ - result: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, - }), - id: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.completionToolCallId, - }, - ], - }, - }) - } - - for (const expectation of [...buildOrchestratorNestedParentResumeExpectations()].reverse()) { - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorNestedParentResumeRequest(requestText(req), expectation.requiredSummaries), - }, - response: { - toolCalls: [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT }), - id: "call_orchestrator_nested_parent_completion_002", - }, - ], - }, - }) - } + addChildCompletionFixtures(mock, ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorNestedChildResumeExpectations(), + steps: ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + matches: shouldMatchOrchestratorNestedChildResumeRequest, + finalResult: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + finalToolCallId: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.completionToolCallId, + }) + addResumeFixtures(mock, { + expectations: buildOrchestratorNestedParentResumeExpectations(), + steps: [], + matches: shouldMatchOrchestratorNestedParentResumeRequest, + finalResult: ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + finalToolCallId: "call_orchestrator_nested_parent_completion_002", + }) mock.addFixture({ match: { @@ -354,7 +248,7 @@ export function addOrchestratorFixtures(mock: InstanceType) { question: `Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer}" to recover the cancelled child.`, follow_up: [{ text: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer }], }), - id: "call_orchestrator_cancellation_child_followup_001", + id: ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, }, ], }, @@ -376,24 +270,11 @@ export function addOrchestratorFixtures(mock: InstanceType) { }, }) - for (const expectation of [...buildOrchestratorCancellationRecoveryResumeExpectations()].reverse()) { - mock.addFixture({ - match: { - predicate: (req: ChatCompletionRequest) => - shouldMatchOrchestratorCancellationRecoveryResumeRequest( - requestText(req), - expectation.requiredSummaries, - ), - }, - response: { - toolCalls: [ - { - name: "attempt_completion", - arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT }), - id: "call_orchestrator_cancellation_parent_completion_002", - }, - ], - }, - }) - } + addResumeFixtures(mock, { + expectations: buildOrchestratorCancellationRecoveryResumeExpectations(), + steps: [], + matches: shouldMatchOrchestratorCancellationRecoveryResumeRequest, + finalResult: ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + finalToolCallId: "call_orchestrator_cancellation_parent_completion_002", + }) } diff --git a/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts index d2a6133fa8..24c3af8c2e 100644 --- a/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts +++ b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts @@ -53,6 +53,7 @@ export const getResourceDiagnosticsConvergenceIssues = ({ }) } + // A final zero intentionally treats a shrinking task stack as converged, unlike registeredTaskCount. if (final.currentTaskStackLength !== baseline.currentTaskStackLength && final.currentTaskStackLength !== 0) { issues.push({ name: "currentTaskStackLength", diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts index de8af5c7fc..e485fa80b7 100644 --- a/apps/vscode-e2e/src/suite/orchestrator.test.ts +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -1,6 +1,6 @@ import * as assert from "assert" -import { RooCodeEventName, type ClineMessage } from "@roo-code/types" +import { RooCodeEventName, type ClineMessage, type RooCodeAPI, type RooCodeResourceDiagnostics } from "@roo-code/types" import { setDefaultSuiteTimeout } from "./test-utils" import { sleep, waitFor, waitUntilCompleted } from "./utils" @@ -26,11 +26,84 @@ import { ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, } from "../fixtures/orchestrator" +const getDefaultOpenRouterConfiguration = () => { + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + + return { + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "openai/gpt-4.1", + currentApiConfigName: "default", + mode: "ask" as const, + autoApprovalEnabled: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + modeApiConfigs: {}, + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + } +} + +async function restoreDefaultOpenRouterConfiguration(api: RooCodeAPI): Promise { + const configuration = getDefaultOpenRouterConfiguration() + await api.upsertProfile("default", configuration, true) + await api.setConfiguration(configuration) +} + +async function drainTaskStack(api: RooCodeAPI, message: string, maxAttempts = 10): Promise { + let attempts = 0 + while (api.getCurrentTaskStack().length > 0) { + assert.ok(attempts < maxAttempts, `Task stack did not drain within ${maxAttempts} clear attempts`) + attempts += 1 + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, message) + await sleep(100) +} + +async function expectDiagnosticsConverged({ + api, + baseline, + observedChildTaskIds, +}: { + api: RooCodeAPI + baseline: RooCodeResourceDiagnostics + observedChildTaskIds: string[] +}): Promise { + await waitFor(() => { + const final = api.getResourceDiagnostics() + + return ( + getResourceDiagnosticsConvergenceIssues({ + baseline, + final, + observedChildTaskIds, + }).length === 0 + ) + }).catch(() => {}) + + assertResourceDiagnosticsConverged({ + baseline, + final: api.getResourceDiagnostics(), + observedChildTaskIds, + }) +} + suite("Roo Code Orchestrator", function () { setDefaultSuiteTimeout(this) + setup(async () => { + await drainTaskStack(globalThis.api, "Task stack should be empty before orchestrator test") + }) + + suiteTeardown(async () => { + await restoreDefaultOpenRouterConfiguration(globalThis.api) + }) + test("orchestrator parent fans out once and fans in three delegated child summaries", async () => { const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() const says: Record = {} const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] @@ -108,12 +181,12 @@ suite("Roo Code Orchestrator", function () { } finally { api.off(RooCodeEventName.Message, messageHandler) api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) - while (api.getCurrentTaskStack().length > 0) { - await api.clearCurrentTask() - } - await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) - assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after cleanup") - await sleep(100) + await drainTaskStack(api, "Task stack should be empty after cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) } }) @@ -226,36 +299,18 @@ suite("Roo Code Orchestrator", function () { } finally { api.off(RooCodeEventName.Message, messageHandler) api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) - while (api.getCurrentTaskStack().length > 0) { - await api.clearCurrentTask() - } - await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) - assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after cleanup") - - const observedChildTaskIds = delegationCompletions.map(({ childId }) => childId) - await waitFor(() => { - const final = api.getResourceDiagnostics() - - return ( - getResourceDiagnosticsConvergenceIssues({ - baseline: baselineDiagnostics, - final, - observedChildTaskIds, - }).length === 0 - ) - }).catch(() => {}) - - const finalDiagnostics = api.getResourceDiagnostics() - assertResourceDiagnosticsConverged({ + await drainTaskStack(api, "Task stack should be empty after cleanup") + await expectDiagnosticsConverged({ + api, baseline: baselineDiagnostics, - final: finalDiagnostics, - observedChildTaskIds, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), }) } }) test("orchestrator parent fans in through a nested child orchestrator", async () => { const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() const says: Record = {} const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] @@ -386,12 +441,12 @@ suite("Roo Code Orchestrator", function () { } finally { api.off(RooCodeEventName.Message, messageHandler) api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) - while (api.getCurrentTaskStack().length > 0) { - await api.clearCurrentTask() - } - await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) - assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after nested cleanup") - await sleep(100) + await drainTaskStack(api, "Task stack should be empty after nested cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) } }) @@ -525,30 +580,11 @@ suite("Roo Code Orchestrator", function () { } finally { api.off(RooCodeEventName.Message, messageHandler) api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) - while (api.getCurrentTaskStack().length > 0) { - await api.clearCurrentTask() - } - await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) - assert.strictEqual(api.getCurrentTaskStack().length, 0, "Task stack should be empty after recovery cleanup") - - const observedChildTaskIds = delegationCompletions.map(({ childId }) => childId) - await waitFor(() => { - const final = api.getResourceDiagnostics() - - return ( - getResourceDiagnosticsConvergenceIssues({ - baseline: baselineDiagnostics, - final, - observedChildTaskIds, - }).length === 0 - ) - }).catch(() => {}) - - const finalDiagnostics = api.getResourceDiagnostics() - assertResourceDiagnosticsConverged({ + await drainTaskStack(api, "Task stack should be empty after recovery cleanup") + await expectDiagnosticsConverged({ + api, baseline: baselineDiagnostics, - final: finalDiagnostics, - observedChildTaskIds, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), }) } }) diff --git a/packages/types/src/api.ts b/packages/types/src/api.ts index fb9f77b2d5..7a0a900ce4 100644 --- a/packages/types/src/api.ts +++ b/packages/types/src/api.ts @@ -1,7 +1,7 @@ import type { EventEmitter } from "events" import type { Socket } from "net" -import type { RooCodeEvents } from "./events.js" +import { RooCodeEventName, type RooCodeEvents } from "./events.js" import type { RooCodeSettings } from "./global-settings.js" import type { HistoryItem } from "./history.js" import type { ProviderSettingsEntry, ProviderSettings } from "./provider-settings.js" @@ -9,7 +9,8 @@ import type { IpcMessage, IpcServerEvents } from "./ipc.js" export type RooCodeAPIEvents = RooCodeEvents -export type RooCodeResourceDiagnosticEventName = +export type RooCodeResourceDiagnosticEventName = Extract< + `${RooCodeEventName}`, | "message" | "taskCreated" | "taskStarted" @@ -18,6 +19,7 @@ export type RooCodeResourceDiagnosticEventName = | "taskDelegationCompleted" | "taskDelegationResumed" | "taskModeSwitched" +> export interface RooCodeResourceDiagnostics { registeredTaskCount: number diff --git a/src/extension/__tests__/api-resource-diagnostics.spec.ts b/src/extension/__tests__/api-resource-diagnostics.spec.ts index 7fbec9d976..f0b90c14ef 100644 --- a/src/extension/__tests__/api-resource-diagnostics.spec.ts +++ b/src/extension/__tests__/api-resource-diagnostics.spec.ts @@ -91,6 +91,7 @@ describe("API#getResourceDiagnostics", () => { it("tracks registered task count across task creation, completion, and abortion", () => { const completedTask = createTask("completed-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([completedTask.taskId]) sidebarProvider.emit(RooCodeEventName.TaskCreated, completedTask) expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) @@ -100,6 +101,7 @@ describe("API#getResourceDiagnostics", () => { expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) const abortedTask = createTask("aborted-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([abortedTask.taskId]) sidebarProvider.emit(RooCodeEventName.TaskCreated, abortedTask) expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) @@ -108,4 +110,16 @@ describe("API#getResourceDiagnostics", () => { expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) }) + + it("prunes registered tasks that are no longer in the provider task stack", () => { + const evictedTask = createTask("evicted-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([evictedTask.taskId]) + sidebarProvider.emit(RooCodeEventName.TaskCreated, evictedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + sidebarProvider.getCurrentTaskStack.mockReturnValue([]) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + }) }) diff --git a/src/extension/api.ts b/src/extension/api.ts index 7832ce7ce7..907bdfb23a 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -10,6 +10,7 @@ import { type RooCodeAPI, type RooCodeSettings, type RooCodeEvents, + type RooCodeResourceDiagnosticEventName, type RooCodeResourceDiagnostics, type ProviderSettings, type ProviderSettingsEntry, @@ -40,7 +41,7 @@ const RESOURCE_DIAGNOSTIC_EVENTS = [ RooCodeEventName.TaskDelegationCompleted, RooCodeEventName.TaskDelegationResumed, RooCodeEventName.TaskModeSwitched, -] as const +] as const satisfies readonly RooCodeResourceDiagnosticEventName[] export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel @@ -267,6 +268,13 @@ export class API extends EventEmitter implements RooCodeAPI { } public getResourceDiagnostics(): RooCodeResourceDiagnostics { + const activeTaskIds = new Set(this.getCurrentTaskStack()) + for (const taskId of this.registeredTaskIds) { + if (!activeTaskIds.has(taskId)) { + this.registeredTaskIds.delete(taskId) + } + } + const listenerCounts = Object.fromEntries( RESOURCE_DIAGNOSTIC_EVENTS.map((eventName) => [eventName, this.listenerCount(eventName)]), ) as RooCodeResourceDiagnostics["listenerCounts"] @@ -355,6 +363,7 @@ export class API extends EventEmitter implements RooCodeAPI { private registerListeners(provider: ClineProvider) { provider.on(RooCodeEventName.TaskCreated, (task) => { this.registeredTaskIds.add(task.taskId) + const unregisterTask = () => this.registeredTaskIds.delete(task.taskId) // Task Lifecycle @@ -367,7 +376,7 @@ export class API extends EventEmitter implements RooCodeAPI { this.emit(RooCodeEventName.TaskCompleted, task.taskId, tokenUsage, toolUsage, { isSubtask: !!task.parentTaskId, }) - this.registeredTaskIds.delete(task.taskId) + unregisterTask() await this.fileLog( `[${new Date().toISOString()}] taskCompleted -> ${task.taskId} | ${JSON.stringify(tokenUsage, null, 2)} | ${JSON.stringify(toolUsage, null, 2)}\n`, @@ -376,7 +385,7 @@ export class API extends EventEmitter implements RooCodeAPI { task.on(RooCodeEventName.TaskAborted, () => { this.emit(RooCodeEventName.TaskAborted, task.taskId) - this.registeredTaskIds.delete(task.taskId) + unregisterTask() }) task.on(RooCodeEventName.TaskFocused, () => { From 8ffd934acaf44babdb69cd5d68966711241dca24 Mon Sep 17 00:00:00 2001 From: Eason Liang Date: Fri, 31 Jul 2026 03:54:03 +0800 Subject: [PATCH 9/9] fix(api): enforce diagnostic event exhaustiveness --- src/extension/api.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/extension/api.ts b/src/extension/api.ts index 907bdfb23a..cf1e3b6ca6 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -32,16 +32,18 @@ import { openClineInNewTab } from "../activate/registerCommands" import { getCommands } from "../services/command/commands" import { getModels } from "../api/providers/fetchers/modelCache" -const RESOURCE_DIAGNOSTIC_EVENTS = [ - RooCodeEventName.Message, - RooCodeEventName.TaskCreated, - RooCodeEventName.TaskStarted, - RooCodeEventName.TaskCompleted, - RooCodeEventName.TaskAborted, - RooCodeEventName.TaskDelegationCompleted, - RooCodeEventName.TaskDelegationResumed, - RooCodeEventName.TaskModeSwitched, -] as const satisfies readonly RooCodeResourceDiagnosticEventName[] +const RESOURCE_DIAGNOSTIC_EVENT_MAP = { + [RooCodeEventName.Message]: true, + [RooCodeEventName.TaskCreated]: true, + [RooCodeEventName.TaskStarted]: true, + [RooCodeEventName.TaskCompleted]: true, + [RooCodeEventName.TaskAborted]: true, + [RooCodeEventName.TaskDelegationCompleted]: true, + [RooCodeEventName.TaskDelegationResumed]: true, + [RooCodeEventName.TaskModeSwitched]: true, +} satisfies Record + +const RESOURCE_DIAGNOSTIC_EVENTS = Object.keys(RESOURCE_DIAGNOSTIC_EVENT_MAP) as RooCodeResourceDiagnosticEventName[] export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel