From 3361384ac4ff9bc38b7d8bdbab1796b1d8412afc Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 5 Aug 2026 12:17:26 -0700 Subject: [PATCH 1/2] fix(libsy): put judge system prompt in instructions, not messages Signed-off-by: Lin Jia --- crates/libsy/src/algorithms/llm_class.rs | 27 ++++++++++++++++--- .../libsy/src/algorithms/util/escalation.rs | 11 ++++---- crates/libsy/src/algorithms/util/llm_judge.rs | 13 ++++----- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index b9c86b62..814246eb 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -916,7 +916,8 @@ mod tests { use super::*; use switchyard_protocol::{ - LlmClientError, LlmRequest, Metadata, completion_text, text_request, text_response, + ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, completion_text, + text_request, text_response, }; use crate::algorithms::util::llm_judge::Judge; @@ -1000,9 +1001,17 @@ mod tests { self.judge_system_prompts.lock().extend( request .llm_request - .messages + .instructions .first() - .and_then(|message| message.text_content("\n")), + .and_then(|instruction| { + instruction.content.iter().find_map(|b| { + if let ContentBlock::Text { text } = b { + Some(text.clone()) + } else { + None + } + }) + }), ); r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string() } else { @@ -1470,7 +1479,17 @@ mod tests { let judge_request = judge.build_request(&State::default(), &request); assert_eq!(judge_request.llm_request.model, request.llm_request.model); - assert_eq!(judge_request.llm_request.messages.len(), 3); + assert_eq!(judge_request.llm_request.instructions.len(), 1); + assert_eq!(judge_request.llm_request.instructions[0].role, Role::System); + assert_eq!( + judge_request.llm_request.instructions[0].content, + InstructionBlock { + role: Role::System, + content: Message::text(Role::System, judge.contract().system_prompt()).content, + } + .content, + ); + assert_eq!(judge_request.llm_request.messages.len(), 2); let contents = judge_request .llm_request .messages diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 49cd5308..b8387f62 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -375,12 +375,13 @@ mod tests { .push(Message::text(Role::Assistant, "this turn's reply")); let built = judge.build_request(&State::default(), &judged); - // Two messages: the rubric as system, the condensed trajectory as user. - assert_eq!(built.llm_request.messages.len(), 2); - assert_eq!(built.llm_request.messages[0].role, Role::System); - assert_eq!(built.llm_request.messages[1].role, Role::User); + // Rubric in instructions, condensed trajectory as the sole user message. + assert_eq!(built.llm_request.instructions.len(), 1); + assert_eq!(built.llm_request.instructions[0].role, Role::System); + assert_eq!(built.llm_request.messages.len(), 1); + assert_eq!(built.llm_request.messages[0].role, Role::User); assert!( - built.llm_request.messages[1] + built.llm_request.messages[0] .text_content("") .is_some_and(|text| text.contains("Conversation turn 4")) ); diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 90430e5a..3898c44f 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -14,7 +14,7 @@ use async_trait::async_trait; use serde::de::DeserializeOwned; use serde_json::Value; use switchyard_protocol::{ - AggLlmResponse, LlmRequest, Message, OutputParams, Role, completion_text, + AggLlmResponse, InstructionBlock, LlmRequest, Message, OutputParams, Role, completion_text, }; use super::classifier_contract::ClassifierContract; @@ -144,14 +144,15 @@ where type Verdict = D::Verdict; fn build_request(&self, state: &State, request: &Request) -> Request { - let mut messages = self.input.build_messages(state, request); - messages.insert( - 0, - Message::text(Role::System, self.contract.system_prompt().to_string()), - ); + let messages = self.input.build_messages(state, request); Request { llm_request: LlmRequest { model: request.llm_request.model.clone(), + instructions: vec![InstructionBlock { + role: Role::System, + content: Message::text(Role::System, self.contract.system_prompt().to_string()) + .content, + }], messages, output: OutputParams { max_output_tokens: Some(self.runtime.max_output_tokens), From 81c23d5f970a6a65a9ff984216836723964839b3 Mon Sep 17 00:00:00 2001 From: Lin Jia Date: Wed, 5 Aug 2026 12:40:33 -0700 Subject: [PATCH 2/2] fix(libsy): read judge system prompt from instructions in Python binding test Signed-off-by: Lin Jia --- tests/test_libsy_minimal_bindings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index b8f0a25c..b3182cd1 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -100,7 +100,7 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: _, response = await algorithm.run(request_body()) - prompt = judge.calls[0]["messages"][0]["content"][0]["text"] + prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] assert prompt == "Custom capability rubric." assert judge.calls[0]["output"]["response_format"]["json_schema"]["schema"][ "properties"