Skip to content

Commit 5214bde

Browse files
ellismgCopilotstephentoub
authored
Fix CAPI reasoning E2E fixtures (#2181)
* Fix CAPI reasoning E2E fixtures Use gpt-5.4 for tests that explicitly configure high reasoning effort, and advertise that model in the streaming replay fixture. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143 * Fix reasoning effort model mismatch in Node.js, Python, Go, Rust, and Java test suites The same issue from .NET (using models that don't support configurable reasoning with explicit reasoning_effort) existed in the other five SDKs: - setModel tests used gpt-4.1 (non-reasoning) with reasoning_effort=high in Node.js, Python, Go, and Rust — switched to gpt-5.4 in all four - Streaming fidelity tests created sessions without an explicit model; since the shared snapshot was already updated to gpt-5.4 these would have worked, but explicitly specifying model=gpt-5.4 matches the .NET pattern and makes the reasoning-capability requirement self-documenting (Node.js, Python, Go, Rust, Java) - Java has no setModel-with-reasoning test but the streaming fidelity fix still applies Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com> * Isolate Node reasoning model fixture Use a fresh client for the streaming reasoning test so its gpt-5.4 model catalog is not shadowed by the shared client's earlier cached fixture. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143 * Isolate Node reasoning fixture context Give the streaming reasoning test a dedicated harness context so both stdio and in-process transports use its gpt-5.4 model catalog. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143 * Isolate reasoning model switch fixtures Load the gpt-5.4 catalog in dedicated test contexts so model-switch assertions exercise the selected reasoning-capable model across SDKs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143 * Isolate reasoning streaming fixtures Give the Python, Go, and .NET streaming reasoning tests dedicated proxy contexts so cached model catalogs cannot hide gpt-5.4. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com> Copilot-Session: 9b80f6cb-b851-4598-8294-f4431c6f3143
1 parent 9f114ff commit 5214bde

13 files changed

Lines changed: 129 additions & 83 deletions

‎dotnet/test/E2E/SessionE2ETests.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -611,14 +611,17 @@ public async Task Should_Set_Model_On_Existing_Session()
611611
[Fact]
612612
public async Task Should_Set_Model_With_ReasoningEffort()
613613
{
614-
var session = await CreateSessionAsync();
614+
await using var isolatedCtx = await E2ETestContext.CreateAsync();
615+
await isolatedCtx.ConfigureForTestAsync("session", nameof(Should_Set_Model_With_ReasoningEffort));
616+
var isolatedClient = isolatedCtx.CreateClient();
617+
await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient);
615618

616619
var modelChangedTask = TestHelper.GetNextEventOfTypeAsync<SessionModelChangeEvent>(session);
617620

618-
await session.SetModelAsync("gpt-4.1", "high");
621+
await session.SetModelAsync("gpt-5.4", "high");
619622

620623
var modelChanged = await modelChangedTask;
621-
Assert.Equal("gpt-4.1", modelChanged.Data.NewModel);
624+
Assert.Equal("gpt-5.4", modelChanged.Data.NewModel);
622625
Assert.Equal("high", modelChanged.Data.ReasoningEffort);
623626
}
624627

‎dotnet/test/E2E/StreamingFidelityE2ETests.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,12 @@ public async Task Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured(
150150
{
151151
// Verifies that setting ReasoningEffort alongside Streaming=true does not break
152152
// the streaming pipeline — deltas still arrive and complete successfully.
153-
var session = await CreateSessionAsync(new SessionConfig
153+
await using var isolatedCtx = await E2ETestContext.CreateAsync();
154+
await isolatedCtx.ConfigureForTestAsync("streaming_fidelity", nameof(Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured));
155+
var isolatedClient = isolatedCtx.CreateClient();
156+
await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient, new SessionConfig
154157
{
158+
Model = "gpt-5.4",
155159
Streaming = true,
156160
ReasoningEffort = "high",
157161
});
@@ -177,8 +181,6 @@ public async Task Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured(
177181
var messages = await session.GetEventsAsync();
178182
var startEvent = Assert.Single(messages.OfType<SessionStartEvent>());
179183
Assert.Equal("high", startEvent.Data.ReasoningEffort);
180-
181-
await session.DisposeAsync();
182184
}
183185

184186
[Fact]

‎go/internal/e2e/session_e2e_test.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,7 +1047,12 @@ func getSystemMessage(exchange testharness.ParsedHttpExchange) string {
10471047
}
10481048

10491049
func TestSetModelWithReasoningEffortE2E(t *testing.T) {
1050+
t.Run("should set model with reasoningeffort", runSetModelWithReasoningEffortE2E)
1051+
}
1052+
1053+
func runSetModelWithReasoningEffortE2E(t *testing.T) {
10501054
ctx := testharness.NewTestContext(t)
1055+
ctx.ConfigureForTest(t)
10511056
client := ctx.NewClient()
10521057
t.Cleanup(func() { client.ForceStop() })
10531058

@@ -1072,15 +1077,15 @@ func TestSetModelWithReasoningEffortE2E(t *testing.T) {
10721077
}
10731078
})
10741079

1075-
if err := session.SetModel(t.Context(), "gpt-4.1", &copilot.SetModelOptions{ReasoningEffort: copilot.String("high")}); err != nil {
1080+
if err := session.SetModel(t.Context(), "gpt-5.4", &copilot.SetModelOptions{ReasoningEffort: copilot.String("high")}); err != nil {
10761081
t.Fatalf("SetModel returned error: %v", err)
10771082
}
10781083

10791084
select {
10801085
case evt := <-modelChanged:
10811086
md, mdOk := evt.Data.(*copilot.SessionModelChangeData)
1082-
if !mdOk || md.NewModel != "gpt-4.1" {
1083-
t.Errorf("Expected newModel 'gpt-4.1', got %v", evt.Data)
1087+
if !mdOk || md.NewModel != "gpt-5.4" {
1088+
t.Errorf("Expected newModel 'gpt-5.4', got %v", evt.Data)
10841089
}
10851090
if !mdOk || md.ReasoningEffort == nil || *md.ReasoningEffort != "high" {
10861091
t.Errorf("Expected reasoningEffort 'high', got %v", evt.Data)

‎go/internal/e2e/streaming_fidelity_e2e_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -285,12 +285,16 @@ func TestStreamingFidelityE2E(t *testing.T) {
285285
})
286286

287287
t.Run("should emit streaming deltas with reasoning effort configured", func(t *testing.T) {
288-
ctx.ConfigureForTest(t)
288+
reasoningCtx := testharness.NewTestContext(t)
289+
reasoningCtx.ConfigureForTest(t)
290+
reasoningClient := reasoningCtx.NewClient()
291+
t.Cleanup(func() { reasoningClient.ForceStop() })
289292

290293
// Verifies that setting ReasoningEffort alongside Streaming=true does not break
291294
// the streaming pipeline — deltas still arrive and complete successfully.
292-
session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
295+
session, err := reasoningClient.CreateSession(t.Context(), &copilot.SessionConfig{
293296
OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
297+
Model: "gpt-5.4",
294298
Streaming: copilot.Bool(true),
295299
ReasoningEffort: "high",
296300
})

‎java/src/test/java/com/github/copilot/StreamingFidelityTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ void testShouldEmitStreamingDeltasWithReasoningEffortConfigured() throws Excepti
254254
try (CopilotClient client = ctx.createClient()) {
255255
CopilotSession session = client
256256
.createSession(new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL)
257-
.setStreaming(true).setReasoningEffort("high"))
257+
.setModel("gpt-5.4").setStreaming(true).setReasoningEffort("high"))
258258
.get();
259259

260260
List<SessionEvent> events = new ArrayList<>();

‎nodejs/test/e2e/session.e2e.test.ts‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -964,15 +964,21 @@ describe("Send Blocking Behavior", async () => {
964964
expect(event.data.newModel).toBe("gpt-4.1");
965965
});
966966

967-
it("should set model with reasoningEffort", async () => {
968-
await using session = await client.createSession({ onPermissionRequest: approveAll });
967+
describe("reasoning effort model switch (isolated to avoid models cache contamination)", async () => {
968+
const { copilotClient: reasoningClient } = await createSdkTestContext();
969969

970-
const modelChangePromise = getNextEventOfType(session, "session.model_change");
970+
it("should set model with reasoningEffort", async () => {
971+
await using session = await reasoningClient.createSession({
972+
onPermissionRequest: approveAll,
973+
});
971974

972-
await session.setModel("gpt-4.1", { reasoningEffort: "high" });
975+
const modelChangePromise = getNextEventOfType(session, "session.model_change");
973976

974-
const event = await modelChangePromise;
975-
expect(event.data.newModel).toBe("gpt-4.1");
976-
expect(event.data.reasoningEffort).toBe("high");
977+
await session.setModel("gpt-5.4", { reasoningEffort: "high" });
978+
979+
const event = await modelChangePromise;
980+
expect(event.data.newModel).toBe("gpt-5.4");
981+
expect(event.data.reasoningEffort).toBe("high");
982+
});
977983
});
978984
});

‎nodejs/test/e2e/streaming_fidelity.e2e.test.ts‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -145,32 +145,37 @@ describe("Streaming Fidelity", async () => {
145145
await session2.disconnect();
146146
});
147147

148-
it("should emit streaming deltas with reasoning effort configured", async () => {
149-
const session = await client.createSession({
150-
onPermissionRequest: approveAll,
151-
streaming: true,
152-
reasoningEffort: "high",
153-
});
148+
describe("reasoning effort (isolated to avoid models cache contamination)", async () => {
149+
const { copilotClient: reasoningClient } = await createSdkTestContext();
154150

155-
const events: SessionEvent[] = [];
156-
session.on((event) => events.push(event));
151+
it("should emit streaming deltas with reasoning effort configured", async () => {
152+
const session = await reasoningClient.createSession({
153+
onPermissionRequest: approveAll,
154+
model: "gpt-5.4",
155+
streaming: true,
156+
reasoningEffort: "high",
157+
});
157158

158-
await session.sendAndWait({ prompt: "What is 15 * 17?" });
159+
const events: SessionEvent[] = [];
160+
session.on((event) => events.push(event));
159161

160-
const deltaEvents = events.filter((e) => e.type === "assistant.message_delta");
161-
expect(deltaEvents.length).toBeGreaterThanOrEqual(1);
162+
await session.sendAndWait({ prompt: "What is 15 * 17?" });
162163

163-
const assistantEvents = events.filter((e) => e.type === "assistant.message");
164-
expect(assistantEvents.length).toBeGreaterThanOrEqual(1);
165-
const lastAssistant = assistantEvents[assistantEvents.length - 1]!;
166-
expect(lastAssistant.data.content).toContain("255");
164+
const deltaEvents = events.filter((e) => e.type === "assistant.message_delta");
165+
expect(deltaEvents.length).toBeGreaterThanOrEqual(1);
167166

168-
// Verify the session was created with reasoning effort via getMessages
169-
const messages = await session.getEvents();
170-
const startEvent = messages.find((m) => m.type === "session.start");
171-
expect(startEvent).toBeDefined();
172-
expect(startEvent!.data.reasoningEffort).toBe("high");
167+
const assistantEvents = events.filter((e) => e.type === "assistant.message");
168+
expect(assistantEvents.length).toBeGreaterThanOrEqual(1);
169+
const lastAssistant = assistantEvents[assistantEvents.length - 1]!;
170+
expect(lastAssistant.data.content).toContain("255");
173171

174-
await session.disconnect();
172+
// Verify the session was created with reasoning effort via getMessages
173+
const messages = await session.getEvents();
174+
const startEvent = messages.find((m) => m.type === "session.start");
175+
expect(startEvent).toBeDefined();
176+
expect(startEvent!.data.reasoningEffort).toBe("high");
177+
178+
await session.disconnect();
179+
});
175180
});
176181
});

‎python/e2e/test_session_e2e.py‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -679,27 +679,36 @@ async def test_should_set_model_with_reasoning_effort(self, ctx: E2ETestContext)
679679
"""Test that setModel passes reasoningEffort and it appears in the model_change event."""
680680
import asyncio
681681

682-
session = await ctx.client.create_session(
683-
on_permission_request=PermissionHandler.approve_all
684-
)
682+
isolated_ctx = E2ETestContext()
683+
await isolated_ctx.setup()
684+
try:
685+
await isolated_ctx.configure_for_test(
686+
"session", "should_set_model_with_reasoningeffort"
687+
)
688+
session = await isolated_ctx.client.create_session(
689+
on_permission_request=PermissionHandler.approve_all
690+
)
685691

686-
model_change_event = asyncio.get_event_loop().create_future()
692+
model_change_event = asyncio.get_event_loop().create_future()
687693

688-
def on_event(event):
689-
if model_change_event.done():
690-
return
694+
def on_event(event):
695+
if model_change_event.done():
696+
return
691697

692-
match event.data:
693-
case SessionModelChangeData() as data:
694-
model_change_event.set_result(data)
698+
match event.data:
699+
case SessionModelChangeData() as data:
700+
model_change_event.set_result(data)
695701

696-
session.on(on_event)
702+
session.on(on_event)
697703

698-
await session.set_model("gpt-4.1", reasoning_effort="high")
704+
await session.set_model("gpt-5.4", reasoning_effort="high")
699705

700-
data = await asyncio.wait_for(model_change_event, timeout=30)
701-
assert data.new_model == "gpt-4.1"
702-
assert data.reasoning_effort == "high"
706+
data = await asyncio.wait_for(model_change_event, timeout=30)
707+
assert data.new_model == "gpt-5.4"
708+
assert data.reasoning_effort == "high"
709+
await session.disconnect()
710+
finally:
711+
await isolated_ctx.teardown()
703712

704713
async def test_should_accept_blob_attachments(self, ctx: E2ETestContext):
705714
# Write the image to disk so the model can view it

‎python/e2e/test_streaming_fidelity_e2e.py‎

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -155,34 +155,44 @@ async def test_should_not_produce_deltas_after_session_resume_with_streaming_dis
155155
finally:
156156
await new_client.force_stop()
157157

158-
async def test_should_emit_streaming_deltas_with_reasoning_effort_configured(
159-
self, ctx: E2ETestContext
160-
):
158+
async def test_should_emit_streaming_deltas_with_reasoning_effort_configured(self):
161159
"""Streaming + reasoning_effort produces delta events and session.start shows effort."""
162160
from copilot.session_events import SessionStartData
163161

164-
session = await ctx.client.create_session(
165-
on_permission_request=PermissionHandler.approve_all,
166-
streaming=True,
167-
reasoning_effort="high",
168-
)
169-
170-
events = []
171-
session.on(lambda event: events.append(event))
172-
162+
isolated_ctx = E2ETestContext()
163+
await isolated_ctx.setup()
173164
try:
174-
await session.send_and_wait("What is 15 * 17?", timeout=60.0)
175-
176-
delta_events = [e for e in events if e.type.value == "assistant.message_delta"]
177-
assert len(delta_events) >= 1, "Expected delta events with streaming=True"
178-
179-
assistant_events = [e for e in events if e.type.value == "assistant.message"]
180-
assert len(assistant_events) >= 1, "Expected final assistant.message"
165+
await isolated_ctx.configure_for_test(
166+
"streaming_fidelity",
167+
"should_emit_streaming_deltas_with_reasoning_effort_configured",
168+
)
169+
session = await isolated_ctx.client.create_session(
170+
on_permission_request=PermissionHandler.approve_all,
171+
model="gpt-5.4",
172+
streaming=True,
173+
reasoning_effort="high",
174+
)
181175

182-
# Check session.start event (from get_events) has reasoning_effort
183-
all_msgs = await session.get_events()
184-
start_event = next((e for e in all_msgs if isinstance(e.data, SessionStartData)), None)
185-
assert start_event is not None, "Expected session.start event"
186-
assert start_event.data.reasoning_effort == "high"
176+
events = []
177+
session.on(lambda event: events.append(event))
178+
179+
try:
180+
await session.send_and_wait("What is 15 * 17?", timeout=60.0)
181+
182+
delta_events = [e for e in events if e.type.value == "assistant.message_delta"]
183+
assert len(delta_events) >= 1, "Expected delta events with streaming=True"
184+
185+
assistant_events = [e for e in events if e.type.value == "assistant.message"]
186+
assert len(assistant_events) >= 1, "Expected final assistant.message"
187+
188+
# Check session.start event (from get_events) has reasoning_effort
189+
all_msgs = await session.get_events()
190+
start_event = next(
191+
(e for e in all_msgs if isinstance(e.data, SessionStartData)), None
192+
)
193+
assert start_event is not None, "Expected session.start event"
194+
assert start_event.data.reasoning_effort == "high"
195+
finally:
196+
await session.disconnect()
187197
finally:
188-
await session.disconnect()
198+
await isolated_ctx.teardown()

‎rust/tests/e2e/session.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -967,7 +967,7 @@ async fn should_set_model_with_reasoningeffort() {
967967

968968
session
969969
.set_model(
970-
"gpt-4.1",
970+
"gpt-5.4",
971971
Some(SetModelOptions::default().with_reasoning_effort("high")),
972972
)
973973
.await
@@ -976,7 +976,7 @@ async fn should_set_model_with_reasoningeffort() {
976976
let data = event
977977
.typed_data::<SessionModelChangeData>()
978978
.expect("session.model_change data");
979-
assert_eq!(data.new_model, "gpt-4.1");
979+
assert_eq!(data.new_model, "gpt-5.4");
980980
assert_eq!(data.reasoning_effort.as_deref(), Some("high"));
981981

982982
session.disconnect().await.expect("disconnect session");

0 commit comments

Comments
 (0)