diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index bc9ee703b4..27ef7437f7 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -611,14 +611,17 @@ public async Task Should_Set_Model_On_Existing_Session() [Fact] public async Task Should_Set_Model_With_ReasoningEffort() { - var session = await CreateSessionAsync(); + await using var isolatedCtx = await E2ETestContext.CreateAsync(); + await isolatedCtx.ConfigureForTestAsync("session", nameof(Should_Set_Model_With_ReasoningEffort)); + var isolatedClient = isolatedCtx.CreateClient(); + await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient); var modelChangedTask = TestHelper.GetNextEventOfTypeAsync(session); - await session.SetModelAsync("gpt-4.1", "high"); + await session.SetModelAsync("gpt-5.4", "high"); var modelChanged = await modelChangedTask; - Assert.Equal("gpt-4.1", modelChanged.Data.NewModel); + Assert.Equal("gpt-5.4", modelChanged.Data.NewModel); Assert.Equal("high", modelChanged.Data.ReasoningEffort); } diff --git a/dotnet/test/E2E/StreamingFidelityE2ETests.cs b/dotnet/test/E2E/StreamingFidelityE2ETests.cs index 4df9ca4420..bea4760c86 100644 --- a/dotnet/test/E2E/StreamingFidelityE2ETests.cs +++ b/dotnet/test/E2E/StreamingFidelityE2ETests.cs @@ -150,8 +150,12 @@ public async Task Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured( { // Verifies that setting ReasoningEffort alongside Streaming=true does not break // the streaming pipeline — deltas still arrive and complete successfully. - var session = await CreateSessionAsync(new SessionConfig + await using var isolatedCtx = await E2ETestContext.CreateAsync(); + await isolatedCtx.ConfigureForTestAsync("streaming_fidelity", nameof(Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured)); + var isolatedClient = isolatedCtx.CreateClient(); + await using var session = await isolatedCtx.CreateSessionAsync(isolatedClient, new SessionConfig { + Model = "gpt-5.4", Streaming = true, ReasoningEffort = "high", }); @@ -177,8 +181,6 @@ public async Task Should_Emit_Streaming_Deltas_With_Reasoning_Effort_Configured( var messages = await session.GetEventsAsync(); var startEvent = Assert.Single(messages.OfType()); Assert.Equal("high", startEvent.Data.ReasoningEffort); - - await session.DisposeAsync(); } [Fact] diff --git a/go/internal/e2e/session_e2e_test.go b/go/internal/e2e/session_e2e_test.go index 13ed75750f..440a303483 100644 --- a/go/internal/e2e/session_e2e_test.go +++ b/go/internal/e2e/session_e2e_test.go @@ -1047,7 +1047,12 @@ func getSystemMessage(exchange testharness.ParsedHttpExchange) string { } func TestSetModelWithReasoningEffortE2E(t *testing.T) { + t.Run("should set model with reasoningeffort", runSetModelWithReasoningEffortE2E) +} + +func runSetModelWithReasoningEffortE2E(t *testing.T) { ctx := testharness.NewTestContext(t) + ctx.ConfigureForTest(t) client := ctx.NewClient() t.Cleanup(func() { client.ForceStop() }) @@ -1072,15 +1077,15 @@ func TestSetModelWithReasoningEffortE2E(t *testing.T) { } }) - if err := session.SetModel(t.Context(), "gpt-4.1", &copilot.SetModelOptions{ReasoningEffort: copilot.String("high")}); err != nil { + if err := session.SetModel(t.Context(), "gpt-5.4", &copilot.SetModelOptions{ReasoningEffort: copilot.String("high")}); err != nil { t.Fatalf("SetModel returned error: %v", err) } select { case evt := <-modelChanged: md, mdOk := evt.Data.(*copilot.SessionModelChangeData) - if !mdOk || md.NewModel != "gpt-4.1" { - t.Errorf("Expected newModel 'gpt-4.1', got %v", evt.Data) + if !mdOk || md.NewModel != "gpt-5.4" { + t.Errorf("Expected newModel 'gpt-5.4', got %v", evt.Data) } if !mdOk || md.ReasoningEffort == nil || *md.ReasoningEffort != "high" { t.Errorf("Expected reasoningEffort 'high', got %v", evt.Data) diff --git a/go/internal/e2e/streaming_fidelity_e2e_test.go b/go/internal/e2e/streaming_fidelity_e2e_test.go index 189b61bf2f..7f6d4fba8e 100644 --- a/go/internal/e2e/streaming_fidelity_e2e_test.go +++ b/go/internal/e2e/streaming_fidelity_e2e_test.go @@ -285,12 +285,16 @@ func TestStreamingFidelityE2E(t *testing.T) { }) t.Run("should emit streaming deltas with reasoning effort configured", func(t *testing.T) { - ctx.ConfigureForTest(t) + reasoningCtx := testharness.NewTestContext(t) + reasoningCtx.ConfigureForTest(t) + reasoningClient := reasoningCtx.NewClient() + t.Cleanup(func() { reasoningClient.ForceStop() }) // Verifies that setting ReasoningEffort alongside Streaming=true does not break // the streaming pipeline — deltas still arrive and complete successfully. - session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{ + session, err := reasoningClient.CreateSession(t.Context(), &copilot.SessionConfig{ OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + Model: "gpt-5.4", Streaming: copilot.Bool(true), ReasoningEffort: "high", }) diff --git a/java/src/test/java/com/github/copilot/StreamingFidelityTest.java b/java/src/test/java/com/github/copilot/StreamingFidelityTest.java index 631496a8f7..bc4999a16f 100644 --- a/java/src/test/java/com/github/copilot/StreamingFidelityTest.java +++ b/java/src/test/java/com/github/copilot/StreamingFidelityTest.java @@ -254,7 +254,7 @@ void testShouldEmitStreamingDeltasWithReasoningEffortConfigured() throws Excepti try (CopilotClient client = ctx.createClient()) { CopilotSession session = client .createSession(new SessionConfig().setOnPermissionRequest(PermissionHandler.APPROVE_ALL) - .setStreaming(true).setReasoningEffort("high")) + .setModel("gpt-5.4").setStreaming(true).setReasoningEffort("high")) .get(); List events = new ArrayList<>(); diff --git a/nodejs/test/e2e/session.e2e.test.ts b/nodejs/test/e2e/session.e2e.test.ts index 88fdf4c29f..d99a3e392d 100644 --- a/nodejs/test/e2e/session.e2e.test.ts +++ b/nodejs/test/e2e/session.e2e.test.ts @@ -964,15 +964,21 @@ describe("Send Blocking Behavior", async () => { expect(event.data.newModel).toBe("gpt-4.1"); }); - it("should set model with reasoningEffort", async () => { - await using session = await client.createSession({ onPermissionRequest: approveAll }); + describe("reasoning effort model switch (isolated to avoid models cache contamination)", async () => { + const { copilotClient: reasoningClient } = await createSdkTestContext(); - const modelChangePromise = getNextEventOfType(session, "session.model_change"); + it("should set model with reasoningEffort", async () => { + await using session = await reasoningClient.createSession({ + onPermissionRequest: approveAll, + }); - await session.setModel("gpt-4.1", { reasoningEffort: "high" }); + const modelChangePromise = getNextEventOfType(session, "session.model_change"); - const event = await modelChangePromise; - expect(event.data.newModel).toBe("gpt-4.1"); - expect(event.data.reasoningEffort).toBe("high"); + await session.setModel("gpt-5.4", { reasoningEffort: "high" }); + + const event = await modelChangePromise; + expect(event.data.newModel).toBe("gpt-5.4"); + expect(event.data.reasoningEffort).toBe("high"); + }); }); }); diff --git a/nodejs/test/e2e/streaming_fidelity.e2e.test.ts b/nodejs/test/e2e/streaming_fidelity.e2e.test.ts index 17b5222616..98b8eb1884 100644 --- a/nodejs/test/e2e/streaming_fidelity.e2e.test.ts +++ b/nodejs/test/e2e/streaming_fidelity.e2e.test.ts @@ -145,32 +145,37 @@ describe("Streaming Fidelity", async () => { await session2.disconnect(); }); - it("should emit streaming deltas with reasoning effort configured", async () => { - const session = await client.createSession({ - onPermissionRequest: approveAll, - streaming: true, - reasoningEffort: "high", - }); + describe("reasoning effort (isolated to avoid models cache contamination)", async () => { + const { copilotClient: reasoningClient } = await createSdkTestContext(); - const events: SessionEvent[] = []; - session.on((event) => events.push(event)); + it("should emit streaming deltas with reasoning effort configured", async () => { + const session = await reasoningClient.createSession({ + onPermissionRequest: approveAll, + model: "gpt-5.4", + streaming: true, + reasoningEffort: "high", + }); - await session.sendAndWait({ prompt: "What is 15 * 17?" }); + const events: SessionEvent[] = []; + session.on((event) => events.push(event)); - const deltaEvents = events.filter((e) => e.type === "assistant.message_delta"); - expect(deltaEvents.length).toBeGreaterThanOrEqual(1); + await session.sendAndWait({ prompt: "What is 15 * 17?" }); - const assistantEvents = events.filter((e) => e.type === "assistant.message"); - expect(assistantEvents.length).toBeGreaterThanOrEqual(1); - const lastAssistant = assistantEvents[assistantEvents.length - 1]!; - expect(lastAssistant.data.content).toContain("255"); + const deltaEvents = events.filter((e) => e.type === "assistant.message_delta"); + expect(deltaEvents.length).toBeGreaterThanOrEqual(1); - // Verify the session was created with reasoning effort via getMessages - const messages = await session.getEvents(); - const startEvent = messages.find((m) => m.type === "session.start"); - expect(startEvent).toBeDefined(); - expect(startEvent!.data.reasoningEffort).toBe("high"); + const assistantEvents = events.filter((e) => e.type === "assistant.message"); + expect(assistantEvents.length).toBeGreaterThanOrEqual(1); + const lastAssistant = assistantEvents[assistantEvents.length - 1]!; + expect(lastAssistant.data.content).toContain("255"); - await session.disconnect(); + // Verify the session was created with reasoning effort via getMessages + const messages = await session.getEvents(); + const startEvent = messages.find((m) => m.type === "session.start"); + expect(startEvent).toBeDefined(); + expect(startEvent!.data.reasoningEffort).toBe("high"); + + await session.disconnect(); + }); }); }); diff --git a/python/e2e/test_session_e2e.py b/python/e2e/test_session_e2e.py index aed1340f55..b6f173f759 100644 --- a/python/e2e/test_session_e2e.py +++ b/python/e2e/test_session_e2e.py @@ -679,27 +679,36 @@ async def test_should_set_model_with_reasoning_effort(self, ctx: E2ETestContext) """Test that setModel passes reasoningEffort and it appears in the model_change event.""" import asyncio - session = await ctx.client.create_session( - on_permission_request=PermissionHandler.approve_all - ) + isolated_ctx = E2ETestContext() + await isolated_ctx.setup() + try: + await isolated_ctx.configure_for_test( + "session", "should_set_model_with_reasoningeffort" + ) + session = await isolated_ctx.client.create_session( + on_permission_request=PermissionHandler.approve_all + ) - model_change_event = asyncio.get_event_loop().create_future() + model_change_event = asyncio.get_event_loop().create_future() - def on_event(event): - if model_change_event.done(): - return + def on_event(event): + if model_change_event.done(): + return - match event.data: - case SessionModelChangeData() as data: - model_change_event.set_result(data) + match event.data: + case SessionModelChangeData() as data: + model_change_event.set_result(data) - session.on(on_event) + session.on(on_event) - await session.set_model("gpt-4.1", reasoning_effort="high") + await session.set_model("gpt-5.4", reasoning_effort="high") - data = await asyncio.wait_for(model_change_event, timeout=30) - assert data.new_model == "gpt-4.1" - assert data.reasoning_effort == "high" + data = await asyncio.wait_for(model_change_event, timeout=30) + assert data.new_model == "gpt-5.4" + assert data.reasoning_effort == "high" + await session.disconnect() + finally: + await isolated_ctx.teardown() async def test_should_accept_blob_attachments(self, ctx: E2ETestContext): # Write the image to disk so the model can view it diff --git a/python/e2e/test_streaming_fidelity_e2e.py b/python/e2e/test_streaming_fidelity_e2e.py index a82c7f674d..a644acb838 100644 --- a/python/e2e/test_streaming_fidelity_e2e.py +++ b/python/e2e/test_streaming_fidelity_e2e.py @@ -155,34 +155,44 @@ async def test_should_not_produce_deltas_after_session_resume_with_streaming_dis finally: await new_client.force_stop() - async def test_should_emit_streaming_deltas_with_reasoning_effort_configured( - self, ctx: E2ETestContext - ): + async def test_should_emit_streaming_deltas_with_reasoning_effort_configured(self): """Streaming + reasoning_effort produces delta events and session.start shows effort.""" from copilot.session_events import SessionStartData - session = await ctx.client.create_session( - on_permission_request=PermissionHandler.approve_all, - streaming=True, - reasoning_effort="high", - ) - - events = [] - session.on(lambda event: events.append(event)) - + isolated_ctx = E2ETestContext() + await isolated_ctx.setup() try: - await session.send_and_wait("What is 15 * 17?", timeout=60.0) - - delta_events = [e for e in events if e.type.value == "assistant.message_delta"] - assert len(delta_events) >= 1, "Expected delta events with streaming=True" - - assistant_events = [e for e in events if e.type.value == "assistant.message"] - assert len(assistant_events) >= 1, "Expected final assistant.message" + await isolated_ctx.configure_for_test( + "streaming_fidelity", + "should_emit_streaming_deltas_with_reasoning_effort_configured", + ) + session = await isolated_ctx.client.create_session( + on_permission_request=PermissionHandler.approve_all, + model="gpt-5.4", + streaming=True, + reasoning_effort="high", + ) - # Check session.start event (from get_events) has reasoning_effort - all_msgs = await session.get_events() - start_event = next((e for e in all_msgs if isinstance(e.data, SessionStartData)), None) - assert start_event is not None, "Expected session.start event" - assert start_event.data.reasoning_effort == "high" + events = [] + session.on(lambda event: events.append(event)) + + try: + await session.send_and_wait("What is 15 * 17?", timeout=60.0) + + delta_events = [e for e in events if e.type.value == "assistant.message_delta"] + assert len(delta_events) >= 1, "Expected delta events with streaming=True" + + assistant_events = [e for e in events if e.type.value == "assistant.message"] + assert len(assistant_events) >= 1, "Expected final assistant.message" + + # Check session.start event (from get_events) has reasoning_effort + all_msgs = await session.get_events() + start_event = next( + (e for e in all_msgs if isinstance(e.data, SessionStartData)), None + ) + assert start_event is not None, "Expected session.start event" + assert start_event.data.reasoning_effort == "high" + finally: + await session.disconnect() finally: - await session.disconnect() + await isolated_ctx.teardown() diff --git a/rust/tests/e2e/session.rs b/rust/tests/e2e/session.rs index 45932ffdd0..f66c7e7725 100644 --- a/rust/tests/e2e/session.rs +++ b/rust/tests/e2e/session.rs @@ -967,7 +967,7 @@ async fn should_set_model_with_reasoningeffort() { session .set_model( - "gpt-4.1", + "gpt-5.4", Some(SetModelOptions::default().with_reasoning_effort("high")), ) .await @@ -976,7 +976,7 @@ async fn should_set_model_with_reasoningeffort() { let data = event .typed_data::() .expect("session.model_change data"); - assert_eq!(data.new_model, "gpt-4.1"); + assert_eq!(data.new_model, "gpt-5.4"); assert_eq!(data.reasoning_effort.as_deref(), Some("high")); session.disconnect().await.expect("disconnect session"); diff --git a/rust/tests/e2e/streaming_fidelity.rs b/rust/tests/e2e/streaming_fidelity.rs index 920ad695d0..5a21a31d68 100644 --- a/rust/tests/e2e/streaming_fidelity.rs +++ b/rust/tests/e2e/streaming_fidelity.rs @@ -237,6 +237,7 @@ async fn should_emit_streaming_deltas_with_reasoning_effort_configured() { let session = client .create_session( ctx.approve_all_session_config() + .with_model("gpt-5.4") .with_streaming(true) .with_reasoning_effort("high"), ) diff --git a/test/snapshots/session/should_set_model_with_reasoningeffort.yaml b/test/snapshots/session/should_set_model_with_reasoningeffort.yaml index 0e019bdad7..ccf204d2ae 100644 --- a/test/snapshots/session/should_set_model_with_reasoningeffort.yaml +++ b/test/snapshots/session/should_set_model_with_reasoningeffort.yaml @@ -1,5 +1,6 @@ models: - claude-sonnet-4.5 + - gpt-5.4 conversations: - messages: - role: system diff --git a/test/snapshots/streaming_fidelity/should_emit_streaming_deltas_with_reasoning_effort_configured.yaml b/test/snapshots/streaming_fidelity/should_emit_streaming_deltas_with_reasoning_effort_configured.yaml index fd825907f6..e720fc34dc 100644 --- a/test/snapshots/streaming_fidelity/should_emit_streaming_deltas_with_reasoning_effort_configured.yaml +++ b/test/snapshots/streaming_fidelity/should_emit_streaming_deltas_with_reasoning_effort_configured.yaml @@ -1,5 +1,5 @@ models: - - claude-sonnet-4.5 + - gpt-5.4 conversations: - messages: - role: system