feat(plugin): add StageRouter support to the NeMo Relay plugin

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "crates/switchyard-server",
     "crates/switchyard-skill-distillation",
     "crates/switchyard-translation",
+    "crates/switchyard-nemo-relay-plugin",
 ]
 
 [workspace.package]
@@ -26,7 +27,9 @@ rust-version = "1.96.1"
 async-stream = "0.3"
 async-trait = "0.1"
 futures = "0.3"
+futures-timer = "3"
 futures-util = "0.3"
+http = "1"
 httpdate = "1"
 parking_lot = "0.12"
 rand = "0.10"

README.md

@@ -89,6 +89,12 @@ Verify the proxy in another terminal:
 curl http://localhost:4000/health
 ```
 
+NeMo Relay can instead load libsy in process through the native API v2
+`nvidia.switchyard` plugin. Relay performs provider I/O while the plugin drives
+`run_stream` and uses `switchyard-translation` for every request and response.
+See the
+[`switchyard-nemo-relay-plugin` guide](crates/switchyard-nemo-relay-plugin/README.md).
+
 For a complete configuration and a test request, follow
 [Getting Started](docs/getting_started.md).
 
@@ -119,6 +125,7 @@ configured LLM client selects one upstream format.
 - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application
 - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types
 - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation
+- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: in-process routing for NeMo Relay
 
 ## Community
 

crates/libsy/Cargo.toml

@@ -16,6 +16,7 @@ async-trait.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 futures.workspace = true
+futures-timer.workspace = true
 # Metrics-only OTel API: instruments record through the host-installed global
 # meter provider. Pinned to the 0.32 line used across the workspace.
 opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }

crates/libsy/examples/streaming_agent.rs

@@ -7,7 +7,8 @@
 //! offloaded as a [`Step::CallLlm`]. The agent serves it with a *streaming* response
 //! ([`LlmResponse::Stream`]) rather than a buffered one. That stream rides untouched
 //! through the algorithm and returns as [`Step::ReturnToAgent`], where the agent drives it
-//! and prints each [`LlmResponseChunk`] as it arrives — true token streaming end to end.
+//! and prints the normalized [`LlmResponseChunk`]s in each event as they arrive — true
+//! token streaming end to end.
 //! Contrast [`Algorithm::run`], which would aggregate the same stream into one buffered
 //! answer. Run with:
 //!   cargo run -p libsy --example streaming_agent
@@ -40,7 +41,8 @@ fn streaming_response(model: &str, tokens: &[&str]) -> Response {
         reason: Some("stop".to_string()),
     });
 
-    let stream: LlmResponseStream = futures::stream::iter(chunks.into_iter().map(Ok)).boxed();
+    let stream: LlmResponseStream =
+        futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed();
     Response {
         llm_response: LlmResponse::Stream(stream),
         metadata: None,
@@ -81,15 +83,17 @@ async fn main() -> Result<()> {
             }
             Step::ReturnToAgent(response) => match response.llm_response {
                 // The stream reached the agent untouched: print each token as it arrives.
-                LlmResponse::Stream(mut chunks) => {
+                LlmResponse::Stream(mut events) => {
                     print!("agent sees: ");
-                    while let Some(chunk) = chunks.next().await {
-                        let chunk = chunk.map_err(|error| {
+                    while let Some(event) = events.next().await {
+                        let event = event.map_err(|error| {
                             LibsyError::external("reading response stream", error)
                         })?;
-                        if let LlmResponseChunk::TextDelta { text, .. } = chunk {
-                            print!("{text}");
-                            std::io::stdout().flush().ok();
+                        for chunk in event.normalized() {
+                            if let LlmResponseChunk::TextDelta { text, .. } = chunk {
+                                print!("{text}");
+                                std::io::stdout().flush().ok();
+                            }
                         }
                     }
                     println!();