diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2d1819b4e1..7c3545a9b1 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -34,6 +34,10 @@ jobs: - name: Setup Node.js and pnpm if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' uses: ./.github/actions/setup-node-pnpm + - name: Run vscode-e2e unit tests + if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' + run: pnpm --filter @roo-code/vscode-e2e test:unit + - name: Install xvfb if: github.event_name != 'pull_request' || steps.e2e-marker.outputs.cache-hit != 'true' run: sudo apt-get install -y xvfb diff --git a/apps/vscode-e2e/package.json b/apps/vscode-e2e/package.json index 3b1a3fb18f..c5b44786a8 100644 --- a/apps/vscode-e2e/package.json +++ b/apps/vscode-e2e/package.json @@ -5,6 +5,7 @@ "lint": "eslint src --ext=ts --max-warnings=0", "check-types": "tsc -p tsconfig.esm.json --noEmit", "format": "prettier --write src", + "test:unit": "vitest run --config vitest.config.ts", "test:ci": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && pnpm test:run", "test:ci:mock": "pnpm -w bundle && pnpm --filter @roo-code/vscode-webview build && USE_MOCK=true pnpm test:run", "test:record": "AIMOCK_RECORD=true pnpm test:ci", @@ -23,6 +24,7 @@ "dotenv-cli": "11.0.0", "glob": "11.1.0", "mocha": "11.2.2", - "rimraf": "6.0.1" + "rimraf": "6.0.1", + "vitest": "4.1.9" } } diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts new file mode 100644 index 0000000000..71c8320e9d --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.spec.ts @@ -0,0 +1,369 @@ +import { describe, expect, it } from "vitest" + +import { + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, + ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_MARKER, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_MARKER, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + buildOrchestratorCancellationRecoveryResumeExpectations, + buildOrchestratorNestedChildResumeExpectations, + buildOrchestratorNestedParentResumeExpectations, + buildOrchestratorRepeatedResumeExpectations, + buildOrchestratorResumeExpectations, + shouldMatchOrchestratorCancellationChildCompletionRequest, + shouldMatchOrchestratorCancellationChildRequest, + shouldMatchOrchestratorCancellationRecoveryResumeRequest, + shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorNestedChildResumeRequest, + shouldMatchOrchestratorNestedParentResumeRequest, + shouldMatchOrchestratorRepeatedResumeRequest, + shouldMatchOrchestratorResumeRequest, +} from "./orchestrator-plan" + +describe("orchestrator delegation plans", () => { + it("defines the first commit single-round ask/architect/code child sequence", () => { + expect(ORCHESTRATOR_FAN_OUT_PARENT_PROMPT).toContain("ORCHESTRATOR_SINGLE_ROUND_FAN_OUT") + expect(ORCHESTRATOR_FAN_OUT_CHILD_STEPS).toEqual([ + expect.objectContaining({ + mode: "ask", + summary: "Requirement summary: gather requirements for the reporting workflow.", + }), + expect.objectContaining({ + mode: "architect", + summary: "Design summary: outline a minimal fan-in architecture.", + }), + expect.objectContaining({ + mode: "code", + summary: "Implementation summary: implement the delegated workflow skeleton.", + }), + ]) + }) + + it("builds cumulative parent resume expectations for fan-in matching", () => { + expect(buildOrchestratorResumeExpectations()).toEqual([ + { + stepIndex: 1, + requiredSummaries: ["Requirement summary: gather requirements for the reporting workflow."], + nextMode: "architect", + }, + { + stepIndex: 2, + requiredSummaries: [ + "Requirement summary: gather requirements for the reporting workflow.", + "Design summary: outline a minimal fan-in architecture.", + ], + nextMode: "code", + }, + { + stepIndex: 3, + requiredSummaries: [ + "Requirement summary: gather requirements for the reporting workflow.", + "Design summary: outline a minimal fan-in architecture.", + "Implementation summary: implement the delegated workflow skeleton.", + ], + nextMode: undefined, + }, + ]) + }) + + it("matches child requests without matching the parent prompt that embeds child markers", () => { + expect( + shouldMatchOrchestratorChildRequest( + "child ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + ), + ).toBe(true) + expect( + shouldMatchOrchestratorChildRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT embeds ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + ), + ).toBe(false) + }) + + it("matches parent resume requests only after cumulative child result injection", () => { + const firstExpectation = buildOrchestratorResumeExpectations()[0]! + const resumeRequest = `ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\n${firstExpectation.requiredSummaries[0]}` + + expect(shouldMatchOrchestratorResumeRequest(resumeRequest, firstExpectation.requiredSummaries)).toBe(true) + expect( + shouldMatchOrchestratorResumeRequest( + `ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\nmissing ${firstExpectation.requiredSummaries[0]}`, + firstExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorResumeRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT without result", + firstExpectation.requiredSummaries, + ), + ).toBe(false) + }) + + it("keeps the final parent result fan-in explicit and reviewable", () => { + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain( + "Requirement summary: gather requirements for the reporting workflow.", + ) + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain("Design summary: outline a minimal fan-in architecture.") + expect(ORCHESTRATOR_FAN_OUT_FINAL_RESULT).toContain( + "Implementation summary: implement the delegated workflow skeleton.", + ) + }) + + it("defines a three-round repeated delegation sequence with ask/architect/code children per round", () => { + expect(ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT).toContain(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS).toHaveLength(9) + expect( + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, mode }) => ({ round, role, mode })), + ).toEqual([ + { round: 1, role: "requirements", mode: "ask" }, + { round: 1, role: "design", mode: "architect" }, + { round: 1, role: "implementation", mode: "code" }, + { round: 2, role: "requirements", mode: "ask" }, + { round: 2, role: "design", mode: "architect" }, + { round: 2, role: "implementation", mode: "code" }, + { round: 3, role: "requirements", mode: "ask" }, + { round: 3, role: "design", mode: "architect" }, + { round: 3, role: "implementation", mode: "code" }, + ]) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]?.summary).toBe( + "Round 1 requirements summary: capture reporting workflow constraints.", + ) + expect(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[8]?.summary).toBe( + "Round 3 implementation summary: validate repeated delegation convergence.", + ) + }) + + it("builds cumulative repeated delegation resume expectations across all rounds", () => { + const expectations = buildOrchestratorRepeatedResumeExpectations() + + expect(expectations).toHaveLength(9) + expect(expectations[0]).toEqual({ + stepIndex: 1, + requiredSummaries: ["Round 1 requirements summary: capture reporting workflow constraints."], + nextMode: "architect", + }) + expect(expectations[2]).toEqual({ + stepIndex: 3, + requiredSummaries: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.slice(0, 3).map(({ summary }) => summary), + nextMode: "ask", + }) + expect(expectations[8]).toEqual({ + stepIndex: 9, + requiredSummaries: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ summary }) => summary), + nextMode: undefined, + }) + }) + + it("matches repeated delegation parent resumes only for cumulative child result injection", () => { + const expectations = buildOrchestratorRepeatedResumeExpectations() + const thirdExpectation = expectations[2]! + const resumeRequest = `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${thirdExpectation.requiredSummaries.join(" completed.\\n\\nResult:\\n")}` + + expect(shouldMatchOrchestratorRepeatedResumeRequest(resumeRequest, thirdExpectation.requiredSummaries)).toBe( + true, + ) + expect( + shouldMatchOrchestratorRepeatedResumeRequest( + `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER} completed.\\n\\nResult:\\nmissing ${thirdExpectation.requiredSummaries[2]}`, + thirdExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorRepeatedResumeRequest( + "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT completed.\\n\\nResult:\\nRound 1 requirements summary: capture reporting workflow constraints.", + ["Round 1 requirements summary: capture reporting workflow constraints."], + ), + ).toBe(false) + }) + + it("composes a repeated delegation final result with every round and child summary", () => { + for (const { round, role, summary } of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { + expect(ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT).toContain(`Round ${round} ${role}: ${summary}`) + } + }) + + it("defines a nested parent/child orchestrator plan with A/B/C/D roles, modes, and summaries", () => { + expect(ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT).toContain(ORCHESTRATOR_NESTED_DELEGATION_MARKER) + expect(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP).toEqual( + expect.objectContaining({ + role: "child-orchestrator", + mode: "orchestrator", + summary: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + }), + ) + expect(ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS).toEqual([ + expect.objectContaining({ + role: "requirements", + mode: "ask", + summary: "Nested requirement summary: capture child orchestrator requirements.", + }), + expect.objectContaining({ + role: "implementation", + mode: "code", + summary: "Nested implementation summary: produce child orchestrator implementation notes.", + }), + ]) + }) + + it("matches nested child orchestrator resumes only after cumulative C/D result injection", () => { + const expectations = buildOrchestratorNestedChildResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: ["Nested requirement summary: capture child orchestrator requirements."], + nextMode: "code", + }, + { + stepIndex: 2, + requiredSummaries: ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.map(({ summary }) => summary), + nextMode: undefined, + }, + ]) + + const secondExpectation = expectations[1]! + const childResumeRequest = `${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker} completed.\\n\\nResult:\\n${secondExpectation.requiredSummaries.join(" completed.\\n\\nResult:\\n")}` + + expect( + shouldMatchOrchestratorNestedChildResumeRequest(childResumeRequest, secondExpectation.requiredSummaries), + ).toBe(true) + expect( + shouldMatchOrchestratorNestedChildResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker} completed.\\n\\nResult:\\nmissing ${secondExpectation.requiredSummaries[1]}`, + secondExpectation.requiredSummaries, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorNestedChildResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${secondExpectation.requiredSummaries[0]}`, + [secondExpectation.requiredSummaries[0]!], + ), + ).toBe(false) + }) + + it("matches top-level nested parent resumes only after the B nested result", () => { + const expectations = buildOrchestratorNestedParentResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: [ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT], + nextMode: undefined, + }, + ]) + + const parentResumeRequest = `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` + expect( + shouldMatchOrchestratorNestedParentResumeRequest(parentResumeRequest, [ + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ]), + ).toBe(true) + expect( + shouldMatchOrchestratorNestedParentResumeRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary}`, + [ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT], + ), + ).toBe(false) + }) + + it("composes nested child and final parent results with the nested summaries", () => { + for (const { summary } of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { + expect(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT).toContain(summary) + } + expect(ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT).toContain(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + }) + + it("defines a cancellation recovery parent/child plan with a pending ask child", () => { + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT).toContain(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP).toEqual( + expect.objectContaining({ + mode: "ask", + role: "cancellation-child", + summary: "Cancellation recovery child summary: resumed after interruption.", + }), + ) + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain("Ask the user exactly") + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt).toContain( + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ) + }) + + it("matches cancellation child requests without colliding with other orchestrator scenarios", () => { + expect( + shouldMatchOrchestratorCancellationChildRequest( + `child ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`, + ), + ).toBe(true) + expect(shouldMatchOrchestratorCancellationChildRequest(ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT)).toBe( + false, + ) + expect( + shouldMatchOrchestratorCancellationChildRequest( + `${ORCHESTRATOR_NESTED_DELEGATION_MARKER} embeds ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker}`, + ), + ).toBe(false) + }) + + it("matches cancellation child completion only after the recovery answer", () => { + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID} ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, + ), + ).toBe(true) + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker} without the answer`, + ), + ).toBe(false) + expect( + shouldMatchOrchestratorCancellationChildCompletionRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} call_cancellation_child_followup_001 ${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}`, + ), + ).toBe(false) + }) + + it("matches cancellation recovery parent resumes only after child result injection", () => { + const expectations = buildOrchestratorCancellationRecoveryResumeExpectations() + expect(expectations).toEqual([ + { + stepIndex: 1, + requiredSummaries: [ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary], + nextMode: undefined, + }, + ]) + + const parentResumeRequest = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}` + expect( + shouldMatchOrchestratorCancellationRecoveryResumeRequest(parentResumeRequest, [ + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + ]), + ).toBe(true) + expect( + shouldMatchOrchestratorCancellationRecoveryResumeRequest( + `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER} completed.\\n\\nResult:\\nmissing`, + [ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary], + ), + ).toBe(false) + }) + + it("composes cancellation recovery final result with an explicit cancellation marker", () => { + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain("cancellation recovery") + expect(ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT).toContain( + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + ) + }) +}) diff --git a/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts new file mode 100644 index 0000000000..c0ffba72f7 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator-plan.ts @@ -0,0 +1,312 @@ +export type OrchestratorFanOutMode = "ask" | "architect" | "code" | "orchestrator" + +export type OrchestratorFanOutChildStep = { + readonly mode: OrchestratorFanOutMode + readonly marker: string + readonly prompt: string + readonly summary: string + readonly newTaskToolCallId: string + readonly completionToolCallId: string +} + +export type OrchestratorRepeatedDelegationRole = "requirements" | "design" | "implementation" +export type OrchestratorNestedDelegationRole = "child-orchestrator" | "requirements" | "implementation" +export type OrchestratorCancellationRecoveryRole = "cancellation-child" + +export type OrchestratorRepeatedDelegationChildStep = OrchestratorFanOutChildStep & { + readonly round: number + readonly role: OrchestratorRepeatedDelegationRole +} + +export type OrchestratorNestedDelegationStep = OrchestratorFanOutChildStep & { + readonly role: OrchestratorNestedDelegationRole +} + +export type OrchestratorCancellationRecoveryStep = OrchestratorFanOutChildStep & { + readonly role: OrchestratorCancellationRecoveryRole + readonly followupAnswer: string +} + +export type OrchestratorResumeExpectation = { + readonly stepIndex: number + readonly requiredSummaries: string[] + readonly nextMode?: OrchestratorFanOutMode +} + +export const ORCHESTRATOR_FAN_OUT_MARKER = "ORCHESTRATOR_SINGLE_ROUND_FAN_OUT" +export const ORCHESTRATOR_FAN_OUT_RESULT_INJECTION = "completed.\\n\\nResult:" +export const ORCHESTRATOR_REPEATED_DELEGATION_MARKER = "ORCHESTRATOR_REPEATED_DELEGATION_STRESS" +export const ORCHESTRATOR_NESTED_DELEGATION_MARKER = "ORCHESTRATOR_NESTED_DELEGATION" +export const ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER = "ORCHESTRATOR_CANCELLATION_RECOVERY" + +export const ORCHESTRATOR_FAN_OUT_CHILD_STEPS: readonly OrchestratorFanOutChildStep[] = [ + { + mode: "ask", + marker: "ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_REQUIREMENTS_CHILD: Complete immediately with the exact result "Requirement summary: gather requirements for the reporting workflow."', + summary: "Requirement summary: gather requirements for the reporting workflow.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_001", + completionToolCallId: "call_orchestrator_fan_out_requirements_completion_001", + }, + { + mode: "architect", + marker: "ORCHESTRATOR_SINGLE_ROUND_DESIGN_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_DESIGN_CHILD: Complete immediately with the exact result "Design summary: outline a minimal fan-in architecture."', + summary: "Design summary: outline a minimal fan-in architecture.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_002", + completionToolCallId: "call_orchestrator_fan_out_design_completion_001", + }, + { + mode: "code", + marker: "ORCHESTRATOR_SINGLE_ROUND_IMPLEMENTATION_CHILD", + prompt: 'ORCHESTRATOR_SINGLE_ROUND_IMPLEMENTATION_CHILD: Complete immediately with the exact result "Implementation summary: implement the delegated workflow skeleton."', + summary: "Implementation summary: implement the delegated workflow skeleton.", + newTaskToolCallId: "call_orchestrator_fan_out_parent_new_task_003", + completionToolCallId: "call_orchestrator_fan_out_implementation_completion_001", + }, +] + +export const ORCHESTRATOR_FAN_OUT_PARENT_PROMPT = `${ORCHESTRATOR_FAN_OUT_MARKER}: Delegate exactly three children in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]!.prompt}". After it returns, create an architect-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1]!.prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2]!.prompt}". After all three return, complete with the final fan-in summary containing every child summary.` + +export const ORCHESTRATOR_FAN_OUT_FINAL_RESULT = `Orchestrator fan-in complete:\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]!.summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[1]!.summary}\n- ${ORCHESTRATOR_FAN_OUT_CHILD_STEPS[2]!.summary}` + +const ORCHESTRATOR_REPEATED_DELEGATION_ROLES: ReadonlyArray<{ + readonly role: OrchestratorRepeatedDelegationRole + readonly mode: OrchestratorFanOutMode + readonly summaryByRound: readonly [string, string, string] +}> = [ + { + role: "requirements", + mode: "ask", + summaryByRound: [ + "capture reporting workflow constraints.", + "confirm dashboard stakeholder needs.", + "verify rollout acceptance criteria.", + ], + }, + { + role: "design", + mode: "architect", + summaryByRound: [ + "outline fan-in checkpoints.", + "refine retry-safe orchestration boundaries.", + "document final aggregation shape.", + ], + }, + { + role: "implementation", + mode: "code", + summaryByRound: [ + "stub delegated workflow shell.", + "wire summary collection fixtures.", + "validate repeated delegation convergence.", + ], + }, +] + +export const ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS: readonly OrchestratorRepeatedDelegationChildStep[] = ( + [1, 2, 3] as const +).flatMap((round, roundIndex) => + ORCHESTRATOR_REPEATED_DELEGATION_ROLES.map(({ role, mode, summaryByRound }, roleIndex) => { + const summary = `Round ${round} ${role} summary: ${summaryByRound[roundIndex]!}` + const marker = `ORCHESTRATOR_REPEATED_ROUND_${round}_${role.toUpperCase()}_CHILD` + + return { + round, + role, + mode, + marker, + prompt: `${marker}: Complete immediately with the exact result "${summary}"`, + summary, + newTaskToolCallId: `call_orchestrator_repeated_parent_new_task_${String((round - 1) * 3 + roleIndex + 1).padStart(3, "0")}`, + completionToolCallId: `call_orchestrator_repeated_${role}_round_${round}_completion_001`, + } + }), +) + +export const ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_REPEATED_DELEGATION_MARKER}: Run exactly three rounds. In each round, delegate exactly three children in order: ask-mode requirements, architect-mode design, then code-mode implementation. Use these exact child messages in order: ${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ prompt }) => `"${prompt}"`).join("; ")}. After each child returns, resume the parent before creating the next child. After all nine children return, complete with a final summary containing every round and child summary.` + +export const ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT = `Orchestrator repeated delegation complete:\n${ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ round, role, summary }) => `- Round ${round} ${role}: ${summary}`).join("\n")}` + +export const ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS: readonly OrchestratorNestedDelegationStep[] = [ + { + role: "requirements", + mode: "ask", + marker: "ORCHESTRATOR_NESTED_REQUIREMENTS_GRANDCHILD", + prompt: 'ORCHESTRATOR_NESTED_REQUIREMENTS_GRANDCHILD: Complete immediately with the exact result "Nested requirement summary: capture child orchestrator requirements."', + summary: "Nested requirement summary: capture child orchestrator requirements.", + newTaskToolCallId: "call_orchestrator_nested_child_new_task_001", + completionToolCallId: "call_orchestrator_nested_requirements_completion_001", + }, + { + role: "implementation", + mode: "code", + marker: "ORCHESTRATOR_NESTED_IMPLEMENTATION_GRANDCHILD", + prompt: 'ORCHESTRATOR_NESTED_IMPLEMENTATION_GRANDCHILD: Complete immediately with the exact result "Nested implementation summary: produce child orchestrator implementation notes."', + summary: "Nested implementation summary: produce child orchestrator implementation notes.", + newTaskToolCallId: "call_orchestrator_nested_child_new_task_002", + completionToolCallId: "call_orchestrator_nested_implementation_completion_001", + }, +] + +export const ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT = `Nested child orchestrator complete:\n- C: ${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary}\n- D: ${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.summary}` + +export const ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP: OrchestratorNestedDelegationStep = { + role: "child-orchestrator", + mode: "orchestrator", + marker: "ORCHESTRATOR_NESTED_CHILD_ORCHESTRATOR", + prompt: `ORCHESTRATOR_NESTED_CHILD_ORCHESTRATOR: Delegate exactly two grandchildren in order. First create an ask-mode child with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.prompt}". After it returns, create a code-mode child with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.prompt}". After both grandchildren return, complete with the nested summary containing both grandchild summaries.`, + summary: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + newTaskToolCallId: "call_orchestrator_nested_parent_new_task_001", + completionToolCallId: "call_orchestrator_nested_child_orchestrator_completion_001", +} + +export const ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT = `${ORCHESTRATOR_NESTED_DELEGATION_MARKER}: Delegate exactly one child orchestrator B in orchestrator mode with this exact message: "${ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.prompt}". After B returns, complete with a final top-level summary containing B's nested result.` + +export const ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT = `Nested top-level orchestrator complete:\n- B: ${ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT}` + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER = "resume recovered cancellation child" +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID = + "call_orchestrator_cancellation_child_followup_001" + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP: OrchestratorCancellationRecoveryStep = { + role: "cancellation-child", + mode: "ask", + marker: "ORCH_CANCEL_RECOVERY_CHILD", + prompt: `ORCH_CANCEL_RECOVERY_CHILD: Ask the user exactly this follow-up question: Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}" to recover the cancelled child. After the user answers, complete with the exact result "Cancellation recovery child summary: resumed after interruption."`, + summary: "Cancellation recovery child summary: resumed after interruption.", + followupAnswer: ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + newTaskToolCallId: "call_orchestrator_cancellation_parent_new_task_001", + completionToolCallId: "call_orchestrator_cancellation_child_completion_002", +} + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT = `${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B with this exact message: "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt}". Wait for B to return; do not complete before B returns. After B returns, complete with a final cancellation recovery summary containing B's result.` + +export const ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT = `Orchestrator cancellation recovery complete:\n- B: ${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary}` + +function buildResumeExpectations(steps: readonly OrchestratorFanOutChildStep[]): OrchestratorResumeExpectation[] { + return steps.map((_step, index) => ({ + stepIndex: index + 1, + requiredSummaries: steps.slice(0, index + 1).map(({ summary }) => summary), + nextMode: steps[index + 1]?.mode, + })) +} + +export function buildOrchestratorResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_FAN_OUT_CHILD_STEPS) +} + +export function buildOrchestratorRepeatedResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) +} + +export function buildOrchestratorNestedChildResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations(ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) +} + +export function buildOrchestratorNestedParentResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations([ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP]) +} + +export function buildOrchestratorCancellationRecoveryResumeExpectations(): OrchestratorResumeExpectation[] { + return buildResumeExpectations([ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP]) +} + +export function shouldMatchOrchestratorChildRequest(rawRequest: string, childMarker: string): boolean { + return ( + rawRequest.includes(childMarker) && + !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker) + ) +} + +function shouldMatchOrchestratorCancellationChildBase(rawRequest: string): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.marker) && + !rawRequest.includes(`${ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER}: Delegate exactly one ask-mode child B`) && + !rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + !rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + !rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) + ) +} + +export function shouldMatchOrchestratorCancellationChildRequest(rawRequest: string): boolean { + return ( + shouldMatchOrchestratorCancellationChildBase(rawRequest) && + !rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID) + ) +} + +export function shouldMatchOrchestratorCancellationChildCompletionRequest(rawRequest: string): boolean { + return ( + shouldMatchOrchestratorCancellationChildBase(rawRequest) && + ((rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID) && + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER)) || + rawRequest.includes( + `\\n${ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER}\\n`, + )) + ) +} +function requestContainsResultSummary(rawRequest: string, summary: string): boolean { + const jsonEscapedSummary = summary.replaceAll("\n", "\\n") + + return rawRequest.includes(`Result:\\n${summary}`) || rawRequest.includes(`Result:\\n${jsonEscapedSummary}`) +} + +export function shouldMatchOrchestratorResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_FAN_OUT_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorRepeatedResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_REPEATED_DELEGATION_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorNestedChildResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorNestedParentResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_NESTED_DELEGATION_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} + +export function shouldMatchOrchestratorCancellationRecoveryResumeRequest( + rawRequest: string, + requiredSummaries: readonly string[], +): boolean { + return ( + rawRequest.includes(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER) && + rawRequest.includes(ORCHESTRATOR_FAN_OUT_RESULT_INJECTION) && + requiredSummaries.every((summary) => requestContainsResultSummary(rawRequest, summary)) + ) +} diff --git a/apps/vscode-e2e/src/fixtures/orchestrator.ts b/apps/vscode-e2e/src/fixtures/orchestrator.ts new file mode 100644 index 0000000000..c7d3cb98d4 --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/orchestrator.ts @@ -0,0 +1,280 @@ +import { LLMock } from "@copilotkit/aimock" +import type { ChatCompletionRequest } from "@copilotkit/aimock" + +export * from "./orchestrator-plan" + +import { + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, + ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER, + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_MARKER, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_MARKER, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_MARKER, + buildOrchestratorCancellationRecoveryResumeExpectations, + buildOrchestratorNestedChildResumeExpectations, + buildOrchestratorNestedParentResumeExpectations, + buildOrchestratorRepeatedResumeExpectations, + buildOrchestratorResumeExpectations, + shouldMatchOrchestratorCancellationChildCompletionRequest, + shouldMatchOrchestratorCancellationChildRequest, + shouldMatchOrchestratorCancellationRecoveryResumeRequest, + shouldMatchOrchestratorChildRequest, + shouldMatchOrchestratorNestedChildResumeRequest, + shouldMatchOrchestratorNestedParentResumeRequest, + shouldMatchOrchestratorRepeatedResumeRequest, + shouldMatchOrchestratorResumeRequest, +} from "./orchestrator-plan" + +const requestText = (req: ChatCompletionRequest) => JSON.stringify(req) + +function addChildCompletionFixtures( + mock: InstanceType, + steps: readonly { marker: string; summary: string; completionToolCallId: string }[], +) { + for (const step of steps) { + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorChildRequest(requestText(req), step.marker), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: step.summary }), + id: step.completionToolCallId, + }, + ], + }, + }) + } +} + +function addResumeFixtures( + mock: InstanceType, + config: { + expectations: readonly { stepIndex: number; requiredSummaries: readonly string[] }[] + steps: readonly { mode: string; prompt: string; newTaskToolCallId: string }[] + matches: (rawRequest: string, requiredSummaries: readonly string[]) => boolean + finalResult: string + finalToolCallId: string + }, +) { + // Register most-cumulative expectations first so less-specific predicates cannot shadow later rounds. + for (const expectation of [...config.expectations].reverse()) { + const nextStep = config.steps[expectation.stepIndex] + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + config.matches(requestText(req), expectation.requiredSummaries), + }, + response: { + toolCalls: nextStep + ? [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: nextStep.mode, + message: nextStep.prompt, + }), + id: nextStep.newTaskToolCallId, + }, + ] + : [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: config.finalResult }), + id: config.finalToolCallId, + }, + ], + }, + }) + } +} + +export function addOrchestratorFixtures(mock: InstanceType) { + const firstFanOutStep = ORCHESTRATOR_FAN_OUT_CHILD_STEPS[0]! + const firstRepeatedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[0]! + const firstNestedGrandchildStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]! + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_FAN_OUT_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstFanOutStep.mode, + message: firstFanOutStep.prompt, + }), + id: firstFanOutStep.newTaskToolCallId, + }, + ], + }, + }) + + addChildCompletionFixtures(mock, ORCHESTRATOR_FAN_OUT_CHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorResumeExpectations(), + steps: ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + matches: shouldMatchOrchestratorResumeRequest, + finalResult: ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + finalToolCallId: "call_orchestrator_fan_out_parent_completion_004", + }) + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_REPEATED_DELEGATION_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstRepeatedStep.mode, + message: firstRepeatedStep.prompt, + }), + id: firstRepeatedStep.newTaskToolCallId, + }, + ], + }, + }) + + addChildCompletionFixtures(mock, ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorRepeatedResumeExpectations(), + steps: ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + matches: shouldMatchOrchestratorRepeatedResumeRequest, + finalResult: ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + finalToolCallId: "call_orchestrator_repeated_parent_completion_010", + }) + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_NESTED_DELEGATION_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.mode, + message: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.prompt, + }), + id: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.newTaskToolCallId, + }, + ], + }, + }) + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.marker), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: firstNestedGrandchildStep.mode, + message: firstNestedGrandchildStep.prompt, + }), + id: firstNestedGrandchildStep.newTaskToolCallId, + }, + ], + }, + }) + + addChildCompletionFixtures(mock, ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) + addResumeFixtures(mock, { + expectations: buildOrchestratorNestedChildResumeExpectations(), + steps: ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + matches: shouldMatchOrchestratorNestedChildResumeRequest, + finalResult: ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + finalToolCallId: ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.completionToolCallId, + }) + addResumeFixtures(mock, { + expectations: buildOrchestratorNestedParentResumeExpectations(), + steps: [], + matches: shouldMatchOrchestratorNestedParentResumeRequest, + finalResult: ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + finalToolCallId: "call_orchestrator_nested_parent_completion_002", + }) + + mock.addFixture({ + match: { + userMessage: new RegExp(ORCHESTRATOR_CANCELLATION_RECOVERY_MARKER), + sequenceIndex: 0, + }, + response: { + toolCalls: [ + { + name: "new_task", + arguments: JSON.stringify({ + mode: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.mode, + message: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.prompt, + }), + id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.newTaskToolCallId, + }, + ], + }, + }) + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorCancellationChildRequest(requestText(req)), + }, + response: { + toolCalls: [ + { + name: "ask_followup_question", + arguments: JSON.stringify({ + question: `Type "${ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer}" to recover the cancelled child.`, + follow_up: [{ text: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.followupAnswer }], + }), + id: ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_TOOL_CALL_ID, + }, + ], + }, + }) + + mock.addFixture({ + match: { + predicate: (req: ChatCompletionRequest) => + shouldMatchOrchestratorCancellationChildCompletionRequest(requestText(req)), + }, + response: { + toolCalls: [ + { + name: "attempt_completion", + arguments: JSON.stringify({ result: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary }), + id: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.completionToolCallId, + }, + ], + }, + }) + + addResumeFixtures(mock, { + expectations: buildOrchestratorCancellationRecoveryResumeExpectations(), + steps: [], + matches: shouldMatchOrchestratorCancellationRecoveryResumeRequest, + finalResult: ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + finalToolCallId: "call_orchestrator_cancellation_parent_completion_002", + }) +} diff --git a/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts b/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts new file mode 100644 index 0000000000..9d9827a27d --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/resource-diagnostics.spec.ts @@ -0,0 +1,74 @@ +import type { RooCodeResourceDiagnosticEventName, RooCodeResourceDiagnostics } from "@roo-code/types" +import { describe, expect, it } from "vitest" + +import { assertResourceDiagnosticsConverged } from "./resource-diagnostics" +type DiagnosticsOverrides = Omit, "listenerCounts"> & { + listenerCounts?: Partial> +} + +const diagnostics = (overrides: DiagnosticsOverrides = {}): RooCodeResourceDiagnostics => ({ + registeredTaskCount: overrides.registeredTaskCount ?? 0, + currentTaskStackLength: overrides.currentTaskStackLength ?? 0, + listenerCounts: { + message: 0, + taskCreated: 0, + taskStarted: 0, + taskCompleted: 0, + taskAborted: 0, + taskDelegationCompleted: 0, + taskDelegationResumed: 0, + taskModeSwitched: 0, + ...overrides.listenerCounts, + }, +}) + +describe("resource diagnostics convergence helper", () => { + it("passes when final diagnostics match the baseline after cleanup", () => { + const baseline = diagnostics() + const final = diagnostics() + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).not.toThrow() + }) + + it("fails when registered task count increases", () => { + const baseline = diagnostics({ registeredTaskCount: 0 }) + const final = diagnostics({ registeredTaskCount: 1 }) + + expect(() => + assertResourceDiagnosticsConverged({ + baseline, + final, + observedChildTaskIds: ["child-1"], + }), + ).toThrow(/registeredTaskCount.*child-1/) + }) + + it("fails when listener counts increase", () => { + const baseline = diagnostics({ listenerCounts: { taskDelegationCompleted: 1 } }) + const final = diagnostics({ listenerCounts: { taskDelegationCompleted: 2 } }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow( + /listenerCounts\.taskDelegationCompleted/, + ) + }) + + it("fails when current task stack length has not converged", () => { + const baseline = diagnostics({ currentTaskStackLength: 0 }) + const final = diagnostics({ currentTaskStackLength: 1 }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow(/currentTaskStackLength/) + }) + + it("includes every leaking counter name in the error message", () => { + const baseline = diagnostics() + const final = diagnostics({ + registeredTaskCount: 2, + currentTaskStackLength: 1, + listenerCounts: { taskCompleted: 1, taskDelegationCompleted: 1 }, + }) + + expect(() => assertResourceDiagnosticsConverged({ baseline, final })).toThrow( + /registeredTaskCount[\s\S]*currentTaskStackLength[\s\S]*listenerCounts\.taskCompleted[\s\S]*listenerCounts\.taskDelegationCompleted/, + ) + }) +}) diff --git a/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts new file mode 100644 index 0000000000..24c3af8c2e --- /dev/null +++ b/apps/vscode-e2e/src/fixtures/resource-diagnostics.ts @@ -0,0 +1,85 @@ +import type { RooCodeResourceDiagnosticEventName, RooCodeResourceDiagnostics } from "@roo-code/types" + +export type ResourceDiagnosticsConvergenceOptions = { + baseline: RooCodeResourceDiagnostics + final: RooCodeResourceDiagnostics + observedChildTaskIds?: string[] +} + +type DiagnosticIssue = { + name: string + baseline: number + final: number +} + +const formatIssue = ({ name, baseline, final }: DiagnosticIssue) => `${name}: baseline=${baseline}, final=${final}` + +const listenerCountIssues = ( + baseline: RooCodeResourceDiagnostics, + final: RooCodeResourceDiagnostics, +): DiagnosticIssue[] => { + const listenerNames = new Set([ + ...(Object.keys(baseline.listenerCounts) as RooCodeResourceDiagnosticEventName[]), + ...(Object.keys(final.listenerCounts) as RooCodeResourceDiagnosticEventName[]), + ]) + + return [...listenerNames].sort().flatMap((listenerName) => { + const baselineCount = baseline.listenerCounts[listenerName] ?? 0 + const finalCount = final.listenerCounts[listenerName] ?? 0 + + return finalCount === baselineCount + ? [] + : [ + { + name: `listenerCounts.${listenerName}`, + baseline: baselineCount, + final: finalCount, + }, + ] + }) +} + +export const getResourceDiagnosticsConvergenceIssues = ({ + baseline, + final, +}: ResourceDiagnosticsConvergenceOptions): DiagnosticIssue[] => { + const issues: DiagnosticIssue[] = [] + + if (final.registeredTaskCount !== baseline.registeredTaskCount) { + issues.push({ + name: "registeredTaskCount", + baseline: baseline.registeredTaskCount, + final: final.registeredTaskCount, + }) + } + + // A final zero intentionally treats a shrinking task stack as converged, unlike registeredTaskCount. + if (final.currentTaskStackLength !== baseline.currentTaskStackLength && final.currentTaskStackLength !== 0) { + issues.push({ + name: "currentTaskStackLength", + baseline: baseline.currentTaskStackLength, + final: final.currentTaskStackLength, + }) + } + + issues.push(...listenerCountIssues(baseline, final)) + + return issues +} + +export const assertResourceDiagnosticsConverged = (options: ResourceDiagnosticsConvergenceOptions) => { + const issues = getResourceDiagnosticsConvergenceIssues(options) + + if (issues.length === 0) { + return + } + + const staleChildHint = + options.observedChildTaskIds && options.observedChildTaskIds.length > 0 + ? ` Observed child task ids: ${options.observedChildTaskIds.join(", ")}.` + : "" + + throw new Error( + `Resource diagnostics did not converge after orchestrator cleanup: ${issues.map(formatIssue).join("; ")}.${staleChildHint}`, + ) +} diff --git a/apps/vscode-e2e/src/runTest.ts b/apps/vscode-e2e/src/runTest.ts index 482e73e945..bbaab7712a 100644 --- a/apps/vscode-e2e/src/runTest.ts +++ b/apps/vscode-e2e/src/runTest.ts @@ -19,6 +19,7 @@ import { addListFilesResultFixtures } from "./fixtures/list-files" import { addReadFileResultFixtures } from "./fixtures/read-file" import { addSearchFilesResultFixtures } from "./fixtures/search-files" import { addSubtaskFixtures } from "./fixtures/subtasks" +import { addOrchestratorFixtures } from "./fixtures/orchestrator" import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool" import { addWriteToFileResultFixtures } from "./fixtures/write-to-file" @@ -126,6 +127,7 @@ async function main() { addReadFileResultFixtures(mock) addSearchFilesResultFixtures(mock) addSubtaskFixtures(mock) + addOrchestratorFixtures(mock) addUseMcpToolResultFixtures(mock) addWriteToFileResultFixtures(mock) addDeepSeekV4Fixtures(mock) diff --git a/apps/vscode-e2e/src/suite/orchestrator.test.ts b/apps/vscode-e2e/src/suite/orchestrator.test.ts new file mode 100644 index 0000000000..e485fa80b7 --- /dev/null +++ b/apps/vscode-e2e/src/suite/orchestrator.test.ts @@ -0,0 +1,591 @@ +import * as assert from "assert" + +import { RooCodeEventName, type ClineMessage, type RooCodeAPI, type RooCodeResourceDiagnostics } from "@roo-code/types" + +import { setDefaultSuiteTimeout } from "./test-utils" +import { sleep, waitFor, waitUntilCompleted } from "./utils" +import { + assertResourceDiagnosticsConverged, + getResourceDiagnosticsConvergenceIssues, +} from "../fixtures/resource-diagnostics" +import { + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP, + ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT, + ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER, + ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, + ORCHESTRATOR_FAN_OUT_CHILD_STEPS, + ORCHESTRATOR_FAN_OUT_FINAL_RESULT, + ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP, + ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS, + ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS, + ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT, + ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, +} from "../fixtures/orchestrator" + +const getDefaultOpenRouterConfiguration = () => { + const aimockUrl = process.env.AIMOCK_URL + const isRecord = process.env.AIMOCK_RECORD === "true" + + return { + apiProvider: "openrouter" as const, + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, + openRouterModelId: "openai/gpt-4.1", + currentApiConfigName: "default", + mode: "ask" as const, + autoApprovalEnabled: false, + alwaysAllowModeSwitch: false, + alwaysAllowSubtasks: false, + modeApiConfigs: {}, + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), + } +} + +async function restoreDefaultOpenRouterConfiguration(api: RooCodeAPI): Promise { + const configuration = getDefaultOpenRouterConfiguration() + await api.upsertProfile("default", configuration, true) + await api.setConfiguration(configuration) +} + +async function drainTaskStack(api: RooCodeAPI, message: string, maxAttempts = 10): Promise { + let attempts = 0 + while (api.getCurrentTaskStack().length > 0) { + assert.ok(attempts < maxAttempts, `Task stack did not drain within ${maxAttempts} clear attempts`) + attempts += 1 + await api.clearCurrentTask() + } + await waitFor(() => api.getCurrentTaskStack().length === 0).catch(() => {}) + assert.strictEqual(api.getCurrentTaskStack().length, 0, message) + await sleep(100) +} + +async function expectDiagnosticsConverged({ + api, + baseline, + observedChildTaskIds, +}: { + api: RooCodeAPI + baseline: RooCodeResourceDiagnostics + observedChildTaskIds: string[] +}): Promise { + await waitFor(() => { + const final = api.getResourceDiagnostics() + + return ( + getResourceDiagnosticsConvergenceIssues({ + baseline, + final, + observedChildTaskIds, + }).length === 0 + ) + }).catch(() => {}) + + assertResourceDiagnosticsConverged({ + baseline, + final: api.getResourceDiagnostics(), + observedChildTaskIds, + }) +} + +suite("Roo Code Orchestrator", function () { + setDefaultSuiteTimeout(this) + + setup(async () => { + await drainTaskStack(globalThis.api, "Task stack should be empty before orchestrator test") + }) + + suiteTeardown(async () => { + await restoreDefaultOpenRouterConfiguration(globalThis.api) + }) + + test("orchestrator parent fans out once and fans in three delegated child summaries", async () => { + const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_FAN_OUT_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === ORCHESTRATOR_FAN_OUT_CHILD_STEPS.length) + + const childIds = delegationCompletions.map(({ childId }) => childId) + assert.strictEqual(new Set(childIds).size, 3, "Orchestrator should create three distinct child tasks") + assert.deepStrictEqual( + delegationCompletions.map(({ parentId }) => parentId), + [parentTaskId, parentTaskId, parentTaskId], + "Every delegation completed event should point to the orchestrator parent", + ) + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + ORCHESTRATOR_FAN_OUT_CHILD_STEPS.map(({ summary }) => summary), + "Delegation completed events should preserve each child summary in order", + ) + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Parent history item should exist") + assert.strictEqual(parent.status, "completed", "Parent should be completed after final fan-in") + assert.deepStrictEqual(parent.childIds, childIds, "Parent childIds should contain all three children") + + for (const [index, childId] of childIds.entries()) { + const child = await api.getTaskHistoryItem(childId) + assert.ok(child, `Child ${index + 1} history item should exist`) + assert.strictEqual(child.parentTaskId, parentTaskId, `Child ${index + 1} should point back to parent`) + assert.strictEqual(child.status, "completed", `Child ${index + 1} should be completed`) + assert.strictEqual( + child.completionResultSummary, + ORCHESTRATOR_FAN_OUT_CHILD_STEPS[index]!.summary, + `Child ${index + 1} completion summary should be persisted`, + ) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => !!text) + + assert.strictEqual(parentCompletionText, ORCHESTRATOR_FAN_OUT_FINAL_RESULT) + for (const { summary } of ORCHESTRATOR_FAN_OUT_CHILD_STEPS) { + assert.ok(parentCompletionText.includes(summary), `Final parent completion should include ${summary}`) + } + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + await drainTaskStack(api, "Task stack should be empty after cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) + } + }) + + test("orchestrator parent repeats three rounds of delegated child fan-in without stack duplication", async () => { + const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + const parentStackSnapshots: string[][] = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + parentStackSnapshots.push(api.getCurrentTaskStack()) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_REPEATED_DELEGATION_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.length) + + const childIds = delegationCompletions.map(({ childId }) => childId) + assert.strictEqual(childIds.length, 9, "Repeated delegation should create nine child tasks") + assert.strictEqual(new Set(childIds).size, 9, "Repeated delegation should create nine distinct child tasks") + assert.deepStrictEqual( + delegationCompletions.map(({ parentId }) => parentId), + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(() => parentTaskId), + "Every repeated delegation completed event should point to the orchestrator parent", + ) + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS.map(({ summary }) => summary), + "Repeated delegation completed events should preserve every round summary in order", + ) + + for (const [index, snapshot] of parentStackSnapshots.entries()) { + const parentOccurrences = snapshot.filter((taskId) => taskId === parentTaskId).length + assert.ok( + parentOccurrences <= 1, + `Parent should not be duplicated in stack after child ${index + 1} resumes`, + ) + } + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Repeated delegation parent history item should exist") + assert.strictEqual( + parent.status, + "completed", + "Repeated delegation parent should complete after final fan-in", + ) + assert.deepStrictEqual( + parent.childIds, + childIds, + "Parent childIds should contain all nine children in order", + ) + assert.strictEqual(new Set(parent.childIds ?? []).size, 9, "Parent childIds should not contain duplicates") + + for (const [index, childId] of childIds.entries()) { + const expectedStep = ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS[index]! + const child = await api.getTaskHistoryItem(childId) + assert.ok(child, `Repeated delegation child ${index + 1} history item should exist`) + assert.strictEqual(child.parentTaskId, parentTaskId, `Child ${index + 1} should point back to parent`) + assert.strictEqual(child.status, "completed", `Child ${index + 1} should be completed`) + assert.strictEqual( + child.mode, + expectedStep.mode, + `Child ${index + 1} mode should match the repeated plan`, + ) + assert.strictEqual( + child.completionResultSummary, + expectedStep.summary, + `Child ${index + 1} round ${expectedStep.round} ${expectedStep.role} summary should be persisted`, + ) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => !!text) + + assert.strictEqual(parentCompletionText, ORCHESTRATOR_REPEATED_DELEGATION_FINAL_RESULT) + for (const { round, role, summary } of ORCHESTRATOR_REPEATED_DELEGATION_CHILD_STEPS) { + assert.ok( + parentCompletionText.includes(`Round ${round} ${role}: ${summary}`), + `Final parent completion should include round ${round} ${role} summary`, + ) + } + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + await drainTaskStack(api, "Task stack should be empty after cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) + } + }) + + test("orchestrator parent fans in through a nested child orchestrator", async () => { + const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await waitUntilCompleted({ + api, + start: () => + api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_NESTED_DELEGATION_PARENT_PROMPT, + }), + }) + + await waitFor(() => delegationCompletions.length === 3) + + assert.deepStrictEqual( + delegationCompletions.map(({ summary }) => summary), + [ + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[0]!.summary, + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[1]!.summary, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ], + "Nested fan-in events should complete C/D before B completes to A", + ) + + const childOrchestratorCompletion = delegationCompletions.find( + ({ parentId, summary }) => + parentId === parentTaskId && summary === ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + ) + assert.ok(childOrchestratorCompletion, "A should receive B's nested orchestrator completion") + + const childOrchestratorId = childOrchestratorCompletion.childId + const grandchildCompletions = delegationCompletions.filter( + ({ parentId }) => parentId === childOrchestratorId, + ) + const grandchildIds = grandchildCompletions.map(({ childId }) => childId) + assert.deepStrictEqual( + grandchildCompletions.map(({ summary }) => summary), + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.map(({ summary }) => summary), + "C/D completions should resume only B before B resumes A", + ) + assert.ok( + delegationCompletions + .filter(({ summary }) => + ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS.some((step) => step.summary === summary), + ) + .every(({ parentId }) => parentId === childOrchestratorId), + "C/D completion events should not directly resume A", + ) + + const parent = await api.getTaskHistoryItem(parentTaskId) + assert.ok(parent, "Nested parent A history item should exist") + assert.strictEqual(parent.status, "completed", "Nested parent A should complete after B fan-in") + assert.deepStrictEqual(parent.childIds, [childOrchestratorId], "A childIds should contain only B") + + const childOrchestrator = await api.getTaskHistoryItem(childOrchestratorId) + assert.ok(childOrchestrator, "Nested child orchestrator B history item should exist") + assert.strictEqual(childOrchestrator.parentTaskId, parentTaskId, "B parentTaskId should point to A") + assert.strictEqual(childOrchestrator.mode, ORCHESTRATOR_NESTED_DELEGATION_CHILD_ORCHESTRATOR_STEP.mode) + assert.strictEqual(childOrchestrator.status, "completed", "B should complete after C/D fan-in") + assert.deepStrictEqual(childOrchestrator.childIds, grandchildIds, "B childIds should contain C/D") + assert.strictEqual( + childOrchestrator.completionResultSummary, + ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT, + "B completion summary should include nested C/D result", + ) + + for (const [index, grandchildId] of grandchildIds.entries()) { + const expectedStep = ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS[index]! + const grandchild = await api.getTaskHistoryItem(grandchildId) + assert.ok(grandchild, `Nested grandchild ${index + 1} history item should exist`) + assert.strictEqual( + grandchild.parentTaskId, + childOrchestratorId, + `Grandchild ${index + 1} should point to B`, + ) + assert.strictEqual( + grandchild.mode, + expectedStep.mode, + `Grandchild ${index + 1} mode should match the nested plan`, + ) + assert.strictEqual(grandchild.status, "completed", `Grandchild ${index + 1} should be completed`) + assert.strictEqual( + grandchild.completionResultSummary, + expectedStep.summary, + `Grandchild ${index + 1} summary should be persisted`, + ) + } + + const childCompletionText = says[childOrchestratorId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text) => text === ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + assert.strictEqual(childCompletionText, ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT) + for (const { summary } of ORCHESTRATOR_NESTED_DELEGATION_GRANDCHILD_STEPS) { + assert.ok(childCompletionText.includes(summary), `B completion should include nested ${summary}`) + } + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text) => text === ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT) + assert.strictEqual(parentCompletionText, ORCHESTRATOR_NESTED_DELEGATION_FINAL_RESULT) + assert.ok( + parentCompletionText.includes(ORCHESTRATOR_NESTED_DELEGATION_CHILD_FINAL_RESULT), + "Final top-level completion should include B's nested summary", + ) + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + await drainTaskStack(api, "Task stack should be empty after nested cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) + } + }) + + test("orchestrator child cancellation stays delegated until explicit recovery", async () => { + const api = globalThis.api + const baselineDiagnostics = api.getResourceDiagnostics() + const asks: Record = {} + const says: Record = {} + const delegationCompletions: Array<{ parentId: string; childId: string; summary: string }> = [] + + const messageHandler = ({ taskId, message }: { taskId: string; message: ClineMessage }) => { + if (message.type === "ask") { + asks[taskId] = asks[taskId] || [] + asks[taskId].push(message) + } + if (message.type === "say" && message.partial === false) { + says[taskId] = says[taskId] || [] + says[taskId].push(message) + } + } + + const delegationCompletedHandler = (parentId: string, childId: string, summary: string) => { + delegationCompletions.push({ parentId, childId, summary }) + } + + api.on(RooCodeEventName.Message, messageHandler) + api.on(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + + try { + const parentTaskId = await api.startNewTask({ + configuration: { + mode: "orchestrator", + alwaysAllowModeSwitch: true, + alwaysAllowSubtasks: true, + autoApprovalEnabled: true, + enableCheckpoints: false, + }, + text: ORCHESTRATOR_CANCELLATION_RECOVERY_PARENT_PROMPT, + }) + + let childTaskId: string | undefined + await waitFor(() => { + const stack = api.getCurrentTaskStack() + const current = stack[stack.length - 1] + if (current && current !== parentTaskId) { + childTaskId = current + return true + } + return false + }) + + await waitFor(() => asks[childTaskId!]?.some(({ ask }) => ask === "followup") ?? false) + await waitFor(async () => (await api.getTaskApiConversationHistoryLength(childTaskId!)) > 0) + + const delegatedChild = await api.getTaskHistoryItem(childTaskId!) + assert.ok(delegatedChild, "Cancellation child history item should exist after delegation") + assert.strictEqual(delegatedChild.parentTaskId, parentTaskId, "B parentTaskId should point to A") + assert.strictEqual(delegatedChild.mode, ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.mode) + + await api.cancelCurrentTask() + + await waitFor(() => api.getCurrentTaskStack().at(-1) === childTaskId) + await waitFor( + () => asks[childTaskId!]?.some(({ type, ask }) => type === "ask" && ask === "resume_task") ?? false, + ) + + const interruptedChild = await api.getTaskHistoryItem(childTaskId!) + assert.strictEqual(interruptedChild?.status, "interrupted", "B should be interrupted after cancellation") + assert.strictEqual( + interruptedChild?.parentTaskId, + parentTaskId, + "Interrupted B should retain its parent link to A", + ) + + const delegatedParent = await api.getTaskHistoryItem(parentTaskId) + assert.strictEqual(delegatedParent?.status, "delegated", "A should stay delegated after B cancellation") + assert.strictEqual(delegatedParent?.awaitingChildId, childTaskId, "A should keep awaiting interrupted B") + assert.strictEqual(delegationCompletions.length, 0, "B cancellation should not emit a completion to A") + assert.strictEqual( + says[parentTaskId]?.find(({ say }) => say === "completion_result"), + undefined, + "A must not receive premature final completion after B cancellation", + ) + + const completedParentTaskId = await waitUntilCompleted({ + api, + start: async () => { + await api.sendMessage(ORCHESTRATOR_CANCELLATION_RECOVERY_FOLLOWUP_ANSWER) + return parentTaskId + }, + }) + + assert.strictEqual( + completedParentTaskId, + parentTaskId, + "Explicitly recovered B should report back and complete A", + ) + assert.deepStrictEqual( + delegationCompletions, + [ + { + parentId: parentTaskId, + childId: childTaskId, + summary: ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + }, + ], + "Recovered B should emit exactly one completion to A", + ) + + const recoveredChild = await api.getTaskHistoryItem(childTaskId!) + assert.strictEqual(recoveredChild?.status, "completed", "B should complete after explicit recovery") + assert.strictEqual( + recoveredChild?.completionResultSummary, + ORCHESTRATOR_CANCELLATION_RECOVERY_CHILD_STEP.summary, + "B recovery summary should be persisted", + ) + + const completedParent = await api.getTaskHistoryItem(parentTaskId) + assert.strictEqual(completedParent?.status, "completed", "A should complete after explicit recovery") + assert.deepStrictEqual(completedParent?.childIds, [childTaskId], "A childIds should contain only B") + + const parentCompletionText = says[parentTaskId] + ?.filter(({ say }) => say === "completion_result") + .map(({ text }) => text?.trim()) + .find((text): text is string => text === ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT) + assert.strictEqual(parentCompletionText, ORCHESTRATOR_CANCELLATION_RECOVERY_FINAL_RESULT) + assert.ok( + parentCompletionText.includes("cancellation recovery"), + "A final summary should explicitly identify cancellation recovery", + ) + } finally { + api.off(RooCodeEventName.Message, messageHandler) + api.off(RooCodeEventName.TaskDelegationCompleted, delegationCompletedHandler) + await drainTaskStack(api, "Task stack should be empty after recovery cleanup") + await expectDiagnosticsConverged({ + api, + baseline: baselineDiagnostics, + observedChildTaskIds: delegationCompletions.map(({ childId }) => childId), + }) + } + }) +}) diff --git a/apps/vscode-e2e/vitest.config.ts b/apps/vscode-e2e/vitest.config.ts new file mode 100644 index 0000000000..d6ecaf0e42 --- /dev/null +++ b/apps/vscode-e2e/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config" + +export default defineConfig({ + test: { + watch: false, + include: ["src/**/*.spec.ts"], + }, +}) diff --git a/packages/types/src/api.ts b/packages/types/src/api.ts index 89e9c8bc2b..7a0a900ce4 100644 --- a/packages/types/src/api.ts +++ b/packages/types/src/api.ts @@ -1,7 +1,7 @@ import type { EventEmitter } from "events" import type { Socket } from "net" -import type { RooCodeEvents } from "./events.js" +import { RooCodeEventName, type RooCodeEvents } from "./events.js" import type { RooCodeSettings } from "./global-settings.js" import type { HistoryItem } from "./history.js" import type { ProviderSettingsEntry, ProviderSettings } from "./provider-settings.js" @@ -9,6 +9,24 @@ import type { IpcMessage, IpcServerEvents } from "./ipc.js" export type RooCodeAPIEvents = RooCodeEvents +export type RooCodeResourceDiagnosticEventName = Extract< + `${RooCodeEventName}`, + | "message" + | "taskCreated" + | "taskStarted" + | "taskCompleted" + | "taskAborted" + | "taskDelegationCompleted" + | "taskDelegationResumed" + | "taskModeSwitched" +> + +export interface RooCodeResourceDiagnostics { + registeredTaskCount: number + currentTaskStackLength: number + listenerCounts: Record +} + export interface RooCodeAPI extends EventEmitter { /** * Starts a new task with an optional initial message and images. @@ -56,6 +74,10 @@ export interface RooCodeAPI extends EventEmitter { * @returns An array of task IDs. */ getCurrentTaskStack(): string[] + /** + * Returns stable task resource counters for test diagnostics. + */ + getResourceDiagnostics(): RooCodeResourceDiagnostics /** * Clears the current task. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c3dd070ac..1e7aa854fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,9 @@ importers: rimraf: specifier: 6.0.1 version: 6.0.1 + vitest: + specifier: 4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@22.20.1)(@vitest/coverage-v8@4.1.9)(@vitest/ui@4.1.9)(jsdom@26.1.0)(vite@8.1.0(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.22.4)(yaml@2.9.0)) apps/vscode-nightly: devDependencies: diff --git a/src/extension/__tests__/api-resource-diagnostics.spec.ts b/src/extension/__tests__/api-resource-diagnostics.spec.ts new file mode 100644 index 0000000000..f0b90c14ef --- /dev/null +++ b/src/extension/__tests__/api-resource-diagnostics.spec.ts @@ -0,0 +1,125 @@ +import { EventEmitter } from "events" + +import { describe, expect, it, vi, beforeEach, type Mock } from "vitest" +import * as vscode from "vscode" + +import { RooCodeEventName } from "@roo-code/types" + +import { API } from "../api" +import type { ClineProvider } from "../../core/webview/ClineProvider" + +type ProviderDouble = EventEmitter & { + context: vscode.ExtensionContext + getCurrentTaskStack: Mock<() => string[]> + on: EventEmitter["on"] +} + +type TaskDouble = EventEmitter & { + taskId: string + parentTaskId?: string + approveAsk: Mock<() => void> + handleWebviewAskResponse: Mock<(response: "messageResponse", text?: string, images?: string[]) => void> +} + +function asClineProvider(provider: ProviderDouble): ClineProvider { + // ClineProvider has private members, so a structural test double requires an unknown bridge. + return provider as unknown as ClineProvider +} + +function createProvider(): ProviderDouble { + const provider = new EventEmitter() as ProviderDouble + provider.context = {} as vscode.ExtensionContext + provider.getCurrentTaskStack = vi.fn().mockReturnValue([]) + return provider +} + +function createTask(taskId: string): TaskDouble { + const task = new EventEmitter() as TaskDouble + task.taskId = taskId + task.approveAsk = vi.fn() + task.handleWebviewAskResponse = vi.fn() + return task +} + +describe("API#getResourceDiagnostics", () => { + let api: API + let sidebarProvider: ProviderDouble + + beforeEach(() => { + sidebarProvider = createProvider() + api = new API({ appendLine: vi.fn() } as unknown as vscode.OutputChannel, asClineProvider(sidebarProvider)) + }) + + it("returns initial resource counters and lifecycle listener counts", () => { + sidebarProvider.getCurrentTaskStack.mockReturnValue(["root-task", "child-task"]) + + expect(api.getResourceDiagnostics()).toEqual({ + registeredTaskCount: 0, + currentTaskStackLength: 2, + listenerCounts: { + [RooCodeEventName.Message]: 0, + [RooCodeEventName.TaskCreated]: 0, + [RooCodeEventName.TaskStarted]: 0, + [RooCodeEventName.TaskCompleted]: 0, + [RooCodeEventName.TaskAborted]: 0, + [RooCodeEventName.TaskDelegationCompleted]: 0, + [RooCodeEventName.TaskDelegationResumed]: 0, + [RooCodeEventName.TaskModeSwitched]: 0, + }, + }) + }) + + it("reflects API lifecycle listener counts after on and off", () => { + const messageListener = () => undefined + const completedListener = () => undefined + + api.on(RooCodeEventName.Message, messageListener) + api.on(RooCodeEventName.TaskCompleted, completedListener) + + expect(api.getResourceDiagnostics().listenerCounts).toMatchObject({ + [RooCodeEventName.Message]: 1, + [RooCodeEventName.TaskCompleted]: 1, + }) + + api.off(RooCodeEventName.Message, messageListener) + + expect(api.getResourceDiagnostics().listenerCounts).toMatchObject({ + [RooCodeEventName.Message]: 0, + [RooCodeEventName.TaskCompleted]: 1, + }) + }) + + it("tracks registered task count across task creation, completion, and abortion", () => { + const completedTask = createTask("completed-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([completedTask.taskId]) + sidebarProvider.emit(RooCodeEventName.TaskCreated, completedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + completedTask.emit(RooCodeEventName.TaskCompleted, completedTask.taskId, {}, {}) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + + const abortedTask = createTask("aborted-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([abortedTask.taskId]) + sidebarProvider.emit(RooCodeEventName.TaskCreated, abortedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + abortedTask.emit(RooCodeEventName.TaskAborted) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + }) + + it("prunes registered tasks that are no longer in the provider task stack", () => { + const evictedTask = createTask("evicted-task") + sidebarProvider.getCurrentTaskStack.mockReturnValue([evictedTask.taskId]) + sidebarProvider.emit(RooCodeEventName.TaskCreated, evictedTask) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(1) + + sidebarProvider.getCurrentTaskStack.mockReturnValue([]) + + expect(api.getResourceDiagnostics().registeredTaskCount).toBe(0) + }) +}) diff --git a/src/extension/api.ts b/src/extension/api.ts index 2d0d5a6975..cf1e3b6ca6 100644 --- a/src/extension/api.ts +++ b/src/extension/api.ts @@ -10,6 +10,8 @@ import { type RooCodeAPI, type RooCodeSettings, type RooCodeEvents, + type RooCodeResourceDiagnosticEventName, + type RooCodeResourceDiagnostics, type ProviderSettings, type ProviderSettingsEntry, type TaskEvent, @@ -30,11 +32,25 @@ import { openClineInNewTab } from "../activate/registerCommands" import { getCommands } from "../services/command/commands" import { getModels } from "../api/providers/fetchers/modelCache" +const RESOURCE_DIAGNOSTIC_EVENT_MAP = { + [RooCodeEventName.Message]: true, + [RooCodeEventName.TaskCreated]: true, + [RooCodeEventName.TaskStarted]: true, + [RooCodeEventName.TaskCompleted]: true, + [RooCodeEventName.TaskAborted]: true, + [RooCodeEventName.TaskDelegationCompleted]: true, + [RooCodeEventName.TaskDelegationResumed]: true, + [RooCodeEventName.TaskModeSwitched]: true, +} satisfies Record + +const RESOURCE_DIAGNOSTIC_EVENTS = Object.keys(RESOURCE_DIAGNOSTIC_EVENT_MAP) as RooCodeResourceDiagnosticEventName[] + export class API extends EventEmitter implements RooCodeAPI { private readonly outputChannel: vscode.OutputChannel private readonly sidebarProvider: ClineProvider private readonly context: vscode.ExtensionContext private readonly ipc?: IpcServer + private readonly registeredTaskIds = new Set() private readonly log: (...args: unknown[]) => void private logfile?: string @@ -253,6 +269,25 @@ export class API extends EventEmitter implements RooCodeAPI { return this.sidebarProvider.getCurrentTaskStack() } + public getResourceDiagnostics(): RooCodeResourceDiagnostics { + const activeTaskIds = new Set(this.getCurrentTaskStack()) + for (const taskId of this.registeredTaskIds) { + if (!activeTaskIds.has(taskId)) { + this.registeredTaskIds.delete(taskId) + } + } + + const listenerCounts = Object.fromEntries( + RESOURCE_DIAGNOSTIC_EVENTS.map((eventName) => [eventName, this.listenerCount(eventName)]), + ) as RooCodeResourceDiagnostics["listenerCounts"] + + return { + registeredTaskCount: this.registeredTaskIds.size, + currentTaskStackLength: this.getCurrentTaskStack().length, + listenerCounts, + } + } + public async clearCurrentTask(_lastMessage?: string) { // Legacy finishSubTask removed; clear current by closing active task instance. await this.sidebarProvider.evictCurrentTask() @@ -329,6 +364,9 @@ export class API extends EventEmitter implements RooCodeAPI { private registerListeners(provider: ClineProvider) { provider.on(RooCodeEventName.TaskCreated, (task) => { + this.registeredTaskIds.add(task.taskId) + const unregisterTask = () => this.registeredTaskIds.delete(task.taskId) + // Task Lifecycle task.on(RooCodeEventName.TaskStarted, async () => { @@ -340,6 +378,7 @@ export class API extends EventEmitter implements RooCodeAPI { this.emit(RooCodeEventName.TaskCompleted, task.taskId, tokenUsage, toolUsage, { isSubtask: !!task.parentTaskId, }) + unregisterTask() await this.fileLog( `[${new Date().toISOString()}] taskCompleted -> ${task.taskId} | ${JSON.stringify(tokenUsage, null, 2)} | ${JSON.stringify(toolUsage, null, 2)}\n`, @@ -348,6 +387,7 @@ export class API extends EventEmitter implements RooCodeAPI { task.on(RooCodeEventName.TaskAborted, () => { this.emit(RooCodeEventName.TaskAborted, task.taskId) + unregisterTask() }) task.on(RooCodeEventName.TaskFocused, () => {