Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions crates/libsy-llm-client/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,14 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {

```rust
use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};
use switchyard_protocol::{completion_text, text_request, Context, LlmResponse, Request};
use switchyard_protocol::{ContentBlock, Context, LlmRequest, LlmResponse, Message, Request, Role};

async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<String> {
let request = Request {
llm_request: text_request(None, "Say hello in five words."),
llm_request: LlmRequest {
messages: vec![Message::text(Role::User, "Say hello in five words.")],
..LlmRequest::default()
},
raw_request: None,
metadata: None,
};
Expand All @@ -91,7 +94,15 @@ async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<Str
.await?;

match response.llm_response {
LlmResponse::Agg(agg) => Ok(completion_text(&agg)),
LlmResponse::Agg(agg) => Ok(agg
.outputs
.first()
.and_then(|output| output.content.first())
.and_then(|block| match block {
ContentBlock::Text { text } => Some(text.clone()),
_ => None,
})
.unwrap_or_default()),
LlmResponse::Stream(_) => Err(LlmClientError::InvalidResponse {
source: "expected a buffered response".into(),
}),
Expand All @@ -106,13 +117,16 @@ Set `stream` on the IR request and drive the returned chunk stream:
```rust
use futures_util::StreamExt;
use switchyard_llm_client::TranslatingLlmClient;
use switchyard_protocol::{text_request, Context, LlmResponse, LlmResponseChunk, Request};
use switchyard_protocol::{Context, LlmRequest, LlmResponse, LlmResponseChunk, Message, Request, Role};

async fn stream(
client: &TranslatingLlmClient,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut llm_request = text_request(None, "Count to five.");
llm_request.stream = true;
let llm_request = LlmRequest {
messages: vec![Message::text(Role::User, "Count to five.")],
stream: true,
..LlmRequest::default()
};
let request = Request { llm_request, raw_request: None, metadata: None };

let response = client
Expand Down
27 changes: 26 additions & 1 deletion crates/libsy-llm-client/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -772,13 +772,38 @@ mod tests {
use std::thread::JoinHandle;

use serde_json::json;
use switchyard_protocol::{LlmRequest, completion_text, text_request};
use switchyard_protocol::{AggLlmResponse, ContentBlock, LlmRequest, Message, Role};
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

use super::*;
use crate::backend::HttpBackendConfig;

fn text_request(model: Option<String>, prompt: impl Into<String>) -> LlmRequest {
LlmRequest {
model,
messages: vec![Message::text(Role::User, prompt)],
..LlmRequest::default()
}
}

fn completion_text(response: &AggLlmResponse) -> String {
response
.outputs
.first()
.map(|output| {
output
.content
.iter()
.filter_map(|block| match block {
ContentBlock::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<String>()
})
.unwrap_or_default()
}

fn config(base_url: &str) -> HttpBackendConfig {
HttpBackendConfig {
base_url: base_url.to_string(),
Expand Down
11 changes: 5 additions & 6 deletions crates/libsy/examples/ensemble.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,11 @@ use parking_lot::Mutex;

use tokio::sync::Notify;

mod support;
use support::{completion_text, prompt_text, text_request};
use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmTarget, LlmTargetSet, Result};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use switchyard_protocol::{
Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,
prompt_text, text_request,
};
use switchyard_protocol::{Context, Decision, LlmResponse, Request, Response, RoutedLlmClient};

const CANDIDATE_MODELS: [&str; 3] = [
"nvidia/qwen/qwen3.6-27b",
Expand Down Expand Up @@ -553,12 +552,12 @@ mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, Ordering};

use crate::support::{completion_text, prompt_text, text_request, text_response};
use futures::StreamExt;
use switchyard_libsy::LlmTarget;
use switchyard_protocol::{
LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Response, Role,
RoutedLlmClient, SamplingParams, Signals, ToolChoice, ToolDefinition, completion_text,
prompt_text, text_request, text_response,
RoutedLlmClient, SamplingParams, Signals, ToolChoice, ToolDefinition,
};
use tokio::sync::Semaphore;

Expand Down
7 changes: 3 additions & 4 deletions crates/libsy/examples/research_agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,13 @@
use std::sync::Arc;

use async_trait::async_trait;
mod support;
use support::{completion_text, text_request, text_response};
use switchyard_libsy::{
Algorithm, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier, Result,
TaskClassifierConfig,
};
use switchyard_protocol::{
Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,
text_request, text_response,
};
use switchyard_protocol::{Context, Decision, LlmResponse, Request, Response, RoutedLlmClient};

const CLASSIFIER: &str = "classifier/model";
const STRONG: &str = "strong/model";
Expand Down
6 changes: 3 additions & 3 deletions crates/libsy/examples/research_agent_core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@

use std::sync::Arc;

mod support;
use support::{completion_text, text_request, text_response};
use switchyard_libsy::{
Algorithm, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier, Result,
Step, TaskClassifierConfig,
};
use switchyard_protocol::{
Context, Decision, LlmResponse, Request, Response, completion_text, text_request, text_response,
};
use switchyard_protocol::{Context, Decision, LlmResponse, Request, Response};
use tokio_stream::StreamExt;

const CLASSIFIER: &str = "classifier/model";
Expand Down
5 changes: 3 additions & 2 deletions crates/libsy/examples/streaming_agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ use std::io::Write;
use std::sync::Arc;

use futures::StreamExt;
mod support;
use support::{completion_text, text_request};
use switchyard_libsy::{Algorithm, LibsyError, LlmTarget, LlmTargetSet, Random, Result, Step};
use switchyard_protocol::{
Context, LlmResponse, LlmResponseChunk, LlmResponseStream, Request, Response, completion_text,
text_request,
Context, LlmResponse, LlmResponseChunk, LlmResponseStream, Request, Response,
};

/// The "real" model call the agent makes to fulfill an offloaded promise: a response
Expand Down
57 changes: 57 additions & 0 deletions crates/libsy/examples/support/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

#![allow(dead_code)]

use switchyard_protocol::{
AggLlmResponse, ContentBlock, LlmRequest, Message, ResponseOutput, Role,
};

pub fn text_request(model: Option<String>, prompt: impl Into<String>) -> LlmRequest {
LlmRequest {
model,
messages: vec![Message::text(Role::User, prompt)],
..LlmRequest::default()
}
}

pub fn prompt_text(request: &LlmRequest) -> String {
request
.messages
.iter()
.filter(|message| message.role == Role::User)
.filter_map(|message| message.text_content("\n"))
.collect::<Vec<_>>()
.join("\n")
}

pub fn text_response(model: Option<String>, completion: impl Into<String>) -> AggLlmResponse {
AggLlmResponse {
model,
outputs: vec![ResponseOutput {
role: Role::Assistant,
content: vec![ContentBlock::Text {
text: completion.into(),
}],
stop_reason: None,
}],
..AggLlmResponse::default()
}
}

pub fn completion_text(response: &AggLlmResponse) -> String {
response
.outputs
.first()
.map(|output| {
output
.content
.iter()
.filter_map(|block| match block {
ContentBlock::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<String>()
})
.unwrap_or_default()
}
6 changes: 2 additions & 4 deletions crates/libsy/src/algorithms/fall_through.rs
Original file line number Diff line number Diff line change
Expand Up @@ -353,12 +353,10 @@ mod tests {
use super::*;
use crate::algorithms::util::prompts;
use crate::core::classifier::Classification;
use crate::text::{completion_text, text_request, text_response};
use crate::{SystemPromptProcessor, TargetPrompts};

use switchyard_protocol::{
LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role, completion_text,
text_request, text_response,
};
use switchyard_protocol::{LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role};

#[derive(Debug, thiserror::Error)]
#[error("{0}")]
Expand Down
5 changes: 2 additions & 3 deletions crates/libsy/src/algorithms/llm_class.rs
Original file line number Diff line number Diff line change
Expand Up @@ -915,9 +915,8 @@ mod tests {
use serde_json::Value;

use super::*;
use switchyard_protocol::{
LlmClientError, LlmRequest, Metadata, completion_text, text_request, text_response,
};
use crate::text::{completion_text, text_request, text_response};
use switchyard_protocol::{LlmClientError, LlmRequest, Metadata};

use crate::algorithms::util::llm_judge::Judge;
use crate::core::algorithm::Algorithm;
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy/src/algorithms/noop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use switchyard_protocol::{Context, Decision};
pub struct Noop {}

/// Test decision carrying the inbound model or a fixed placeholder.
pub struct NoopDecision {
pub(crate) struct NoopDecision {
model: String,
}

Expand Down
8 changes: 3 additions & 5 deletions crates/libsy/src/algorithms/passthrough.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ impl Passthrough {
}

/// Decision emitted before [`Passthrough`] calls its configured target.
pub struct PassthroughDecision {
pub(crate) struct PassthroughDecision {
model_id: String,
}

Expand Down Expand Up @@ -77,10 +77,8 @@ mod tests {

use super::Passthrough;
use crate::core::algorithm::{Algorithm, LlmTarget};
use switchyard_protocol::{
Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,
text_request, text_response,
};
use crate::text::{completion_text, text_request, text_response};
use switchyard_protocol::{Context, Decision, LlmResponse, Request, Response, RoutedLlmClient};

/// Echoes the selected target so tests can inspect which target was called.
/// TODO: Duplicated from rand.rs
Expand Down
14 changes: 6 additions & 8 deletions crates/libsy/src/algorithms/rand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,12 @@ use rand::SeedableRng;
use rand::distr::{Distribution, weighted::WeightedIndex};
use rand::rngs::StdRng;

use crate::algorithms::fall_through::{FallThrough, FallThroughDecision};
use crate::algorithms::fall_through::FallThrough;
use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::{LibsyError, Result};
use switchyard_protocol::{Context, Request, Response, RoutedLlmClient};

/// Compatibility name for the decision produced by [`Random`].
pub type RandomDecision = FallThroughDecision;

/// Stateless weighted classifier used by random fall-through routing.
pub struct RandomClassifier {
targets: Vec<String>,
Expand Down Expand Up @@ -181,11 +178,12 @@ mod tests {
use super::*;
use std::collections::HashSet;

use switchyard_protocol::{Metadata, completion_text, text_request, text_response};

use crate::DriverError;
use crate::algorithms::fall_through::FallThroughDecision;
use crate::algorithms::util::affinity::AffinityRouter;
use crate::core::algorithm::LlmTarget;
use crate::text::{completion_text, text_request, text_response};
use switchyard_protocol::Metadata;
use switchyard_protocol::{Decision, LlmResponse, Request, RoutedLlmClient, Signals};

/// Echoes the selected target so tests can inspect which target was called.
Expand Down Expand Up @@ -456,10 +454,10 @@ mod tests {
);
let concrete = decision
.as_any()
.downcast_ref::<RandomDecision>()
.downcast_ref::<FallThroughDecision>()
.ok_or_else(|| {
LibsyError::from(DriverError::TypeMismatch {
expected: "RandomDecision",
expected: "FallThroughDecision",
})
})?;
assert_eq!(concrete.selected_model, "only/model");
Expand Down
3 changes: 2 additions & 1 deletion crates/libsy/src/algorithms/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,11 +223,12 @@ fn build_route(
mod tests {
use std::sync::Arc;

use crate::text::text_response;
use async_trait::async_trait;
use parking_lot::Mutex;
use serde_json::json;
use switchyard_protocol::{
ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, text_response,
ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat,
};

use super::*;
Expand Down
29 changes: 26 additions & 3 deletions crates/libsy/src/algorithms/subagent_affinity_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,34 @@ use super::util::subagent::SubagentOverride;
use crate::Result;
use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::text::{completion_text, text_request, text_response};
use switchyard_protocol::{
Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, completion_text,
slice_to_header_map, text_request, text_response,
Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
};

fn metadata_from_headers(headers: &[(&str, &str)]) -> Metadata {
let mut metadata = Metadata::default();
for (name, value) in headers {
match *name {
"x-claude-code-session-id" | "x-codex-session-id" => {
metadata.session_id = Some((*value).to_string());
}
"x-claude-code-agent-id" => {
metadata.agent_id = Some((*value).to_string());
metadata.is_subagent = true;
metadata.is_delegated_work = true;
}
"x-openai-subagent" => {
metadata.agent_kind = Some((*value).to_string());
metadata.is_subagent = true;
metadata.is_delegated_work = matches!(*value, "collab_spawn" | "review");
}
_ => {}
}
}
metadata
}

/// A client that echoes the routed target name back as the completion.
struct EchoClient;

Expand Down Expand Up @@ -77,7 +100,7 @@ fn request(headers: &[(&str, &str)]) -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
metadata: Some(Metadata::from_headers(&slice_to_header_map(headers))),
metadata: Some(metadata_from_headers(headers)),
}
}

Expand Down
Loading
Loading