From 6816575a34231f6cbad7dcd673c07f3643afc8fc Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 16:46:13 -0600 Subject: [PATCH 01/15] feat(relay): add native routing plugin Signed-off-by: Bryan Bednarski --- Cargo.lock | 186 +++++ Cargo.toml | 1 + .../switchyard-nemo-relay-plugin/Cargo.toml | 31 + .../config.schema.json | 124 +++ .../relay-plugin.toml | 33 + .../scripts/package_bundle.py | 72 ++ .../src/client.rs | 176 ++++ .../src/config.rs | 770 ++++++++++++++++++ .../src/executor.rs | 137 ++++ .../switchyard-nemo-relay-plugin/src/ffi.rs | 444 ++++++++++ .../switchyard-nemo-relay-plugin/src/lib.rs | 401 +++++++++ .../src/runtime.rs | 601 ++++++++++++++ .../src/translation.rs | 116 +++ 13 files changed, 3092 insertions(+) create mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.toml create mode 100644 crates/switchyard-nemo-relay-plugin/config.schema.json create mode 100644 crates/switchyard-nemo-relay-plugin/relay-plugin.toml create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py create mode 100644 crates/switchyard-nemo-relay-plugin/src/client.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/config.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/executor.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/ffi.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/lib.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/translation.rs diff --git a/Cargo.lock b/Cargo.lock index 93cdbf64e..12e851537 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -239,6 +248,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "bumpalo" @@ -287,6 +299,20 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.2" @@ -758,6 +784,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1051,6 +1101,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nemo-relay-plugin" +version = "0.7.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac90070f6fe5778d882b7c0a1457441a698a9d6ab85e9514e9a3fefed87dbc" +dependencies = [ + "nemo-relay-types", + "serde", + "serde_json", +] + +[[package]] +name = "nemo-relay-types" +version = "0.7.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1a46140ebab2df2201d385aad95e62149bd1d4a230efe74df7ec7b594cad24" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1060,6 +1135,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.17.0" @@ -2007,6 +2091,23 @@ dependencies = [ "wiremock", ] +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "switchyard-translation", + "tokio", +] + [[package]] name = "switchyard-protocol" version = "0.2.0" @@ -2452,6 +2553,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2488,6 +2609,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2634,12 +2767,65 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 2325e1ac2..d21605dbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-translation", + "crates/switchyard-nemo-relay-plugin", ] [workspace.package] diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml new file mode 100644 index 000000000..1c94279dd --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +description = "Switchyard-owned HTTP routing plugin for NeMo Relay" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +async-trait.workspace = true +futures-util.workspace = true +http = "1" +# Use the published release candidate during development. This becomes `0.7.0` +# before the integration lands, after the stable crate is published. +nemo-relay-plugin = "=0.7.0-rc.4" +serde.workspace = true +serde_json.workspace = true +switchyard-libsy.workspace = true +switchyard-llm-client.workspace = true +switchyard-protocol.workspace = true +switchyard-translation.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json new file mode 100644 index 000000000..3ac510bfd --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Switchyard NeMo Relay Plugin", + "description": "In-process Switchyard routing with Switchyard-owned provider HTTP dispatch.", + "type": "object", + "additionalProperties": false, + "required": ["version", "algorithm", "targets", "default_targets"], + "properties": { + "version": { + "const": 2, + "description": "Library-only Switchyard configuration version." + }, + "priority": { + "type": "integer", + "default": 0 + }, + "max_retries": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "default": 3, + "description": "Routing retries after the initial libsy run. Every retry starts a fresh run." + }, + "algorithm": { + "description": "Router configuration. This release supports random and capability-based llm_classifier only.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "random" }, + "seed": { "type": ["integer", "null"], "minimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "classifier_target", + "weak_target", + "strong_target", + "base_threshold" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the judge. The target must use openai_chat or openai_responses because the judge requires a JSON-schema response format." + }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "min_confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "capability_elevated_floor": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 1 + }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0 + }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, + "session_affinity": { "type": "boolean", "default": false }, + "message_hash_fallback": { "type": "boolean", "default": false } + } + } + ] + }, + "targets": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["model", "protocol", "base_url"], + "properties": { + "model": { "type": "string", "minLength": 1 }, + "protocol": { + "enum": ["openai_chat", "openai_responses", "anthropic_messages"] + }, + "endpoint": { + "type": "string", + "pattern": "^$|^/", + "description": "Optional provider endpoint override. The resolved URL must end in the canonical route for the selected protocol." + }, + "base_url": { + "type": "string", + "pattern": "^https?://" + }, + "weight": { "type": "number", "minimum": 0, "default": 1 }, + "headers": { + "type": "object", + "description": "Static non-secret provider headers. Credential-bearing headers must use header_env.", + "additionalProperties": { "type": "string" } + }, + "header_env": { + "type": "object", + "description": "Maps provider header names to environment-variable names resolved by the plugin process.", + "additionalProperties": { "type": "string", "minLength": 1 } + } + } + } + }, + "default_targets": { + "type": "object", + "description": "Maps each managed inbound protocol to its trusted fallback target.", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "openai_chat": { "type": "string", "minLength": 1 }, + "openai_responses": { "type": "string", "minLength": 1 }, + "anthropic_messages": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml new file mode 100644 index 000000000..9ba7c2bfc --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 + +[plugin] +id = "nvidia.switchyard" +kind = "rust_dynamic" + +[compat] +# Development lower bound for validation against the published Relay RC. Move +# this to `>=0.7.0,<1.0` together with the SDK dependency before release. +relay = ">=0.7.0-rc.4,<1.0" +native_api = "1" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native", "config_schema"] + +[config_schema] +path = "config.schema.json" + +[source] +artifact = "" + +[integrity] +sha256 = "sha256:" + +[load] +library = "" +symbol = "nemo_relay_register_plugin" diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py new file mode 100644 index 000000000..168cbff4a --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build an operator-facing NeMo Relay plugin bundle from a compiled cdylib.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = CRATE_ROOT.parents[1] + + +def digest(path: Path) -> str: + """Return the lowercase SHA-256 digest for a file.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def main() -> None: + """Materialize a self-contained plugin bundle in an empty directory.""" + parser = argparse.ArgumentParser() + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + library = args.library.resolve() + if not library.is_file(): + parser.error(f"compiled plugin library does not exist: {library}") + + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + placeholders = ("", "") + missing = [placeholder for placeholder in placeholders if placeholder not in manifest] + if missing: + parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}") + + output = args.output.resolve() + if output.exists() and not output.is_dir(): + parser.error(f"bundle output exists and is not a directory: {output}") + if output.is_dir() and any(output.iterdir()): + parser.error(f"bundle output directory must be empty: {output}") + output.mkdir(parents=True, exist_ok=True) + + artifact = output / library.name + shutil.copy2(library, artifact) + for name in ("config.schema.json", "README.md"): + shutil.copy2(CRATE_ROOT / name, output / name) + for name in ("LICENSE", "NOTICE"): + shutil.copy2(REPOSITORY_ROOT / name, output / name) + + artifact_digest = digest(artifact) + manifest = manifest.replace("", artifact.name) + manifest = manifest.replace("", artifact_digest) + (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") + + checksums = [] + for path in sorted(output.iterdir(), key=lambda item: item.name): + if path.name != "SHA256SUMS" and path.is_file(): + checksums.append(f"{digest(path)} {path.name}") + (output / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8") + + print(output) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs new file mode 100644 index 000000000..c8e00a32b --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Switchyard-owned HTTP clients bound to one semantic routing target. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; +use switchyard_protocol::{ + Context, Decision, LlmClientError, Request, Response, RoutedLlmClient, WireFormat, +}; +use switchyard_translation::TranslationEngine; + +use crate::translation; + +/// A provider client bound to one configured Switchyard target. +/// +/// libsy routes with a stable semantic name (for example `fast`). The provider +/// still expects its own model id (for example `meta/llama-3.1-8b-instruct`). +/// Keeping that mapping here prevents an algorithm's semantic labels from +/// leaking into provider requests. +pub(crate) struct TargetClient { + provider_model: String, + target_format: WireFormat, + inner: TranslatingLlmClient, + translation: TranslationEngine, +} + +impl TargetClient { + pub(crate) fn new( + provider_model: String, + target_format: WireFormat, + dispatch_url: String, + headers: BTreeMap, + ) -> Result { + let backend_config = HttpBackendConfig { + // `dispatch_url` is already resolved by configuration. Backend URL + // joining accepts a complete canonical endpoint as well as a base + // URL/prefix. + base_url: dispatch_url, + api_key: None, + extra_headers: headers, + extra_body: BTreeMap::new(), + // Routing retries belong to the plugin: every retry must start a + // fresh libsy run and obtain a fresh decision. + max_retries: 0, + }; + let backend = match target_format { + WireFormat::OpenAiChat => Backend::OpenAiChat(backend_config), + WireFormat::OpenAiResponses => Backend::OpenAiResponses(backend_config), + WireFormat::AnthropicMessages => Backend::Anthropic(backend_config), + }; + let model = ModelConfig::new(provider_model.clone(), backend, None); + let inner = TranslatingLlmClient::new(&[model])?; + Ok(Self { + provider_model, + target_format, + inner, + translation: TranslationEngine::default(), + }) + } + + /// Retargets only the provider-facing transport metadata. + /// + /// Correlation and agent identity remain available to libsy, while inbound + /// HTTP headers are deliberately removed. Provider credentials come solely + /// from this target's `headers` / `header_env` configuration. + fn prepare_request(&self, mut request: Request) -> Request { + let metadata = request.metadata.get_or_insert_default(); + metadata.wire_format = Some(self.target_format); + metadata.http_headers = None; + request + } + + #[cfg(test)] + fn provider_model(&self) -> &str { + &self.provider_model + } +} + +#[async_trait] +impl RoutedLlmClient for TargetClient { + async fn call( + &self, + ctx: Context, + request: Request, + _decision: Arc, + ) -> Result { + let request = self.prepare_request(request); + translation::validate_target_request( + &self.translation, + self.target_format, + &request.llm_request, + ) + .map_err(LlmClientError::RequestEncoding)?; + self.inner + .call_rewrite_model(ctx, request, Some(&self.provider_model)) + .await + } + + fn supports_count_tokens(&self) -> bool { + self.target_format == WireFormat::AnthropicMessages + } + + async fn count_tokens(&self, request: Request) -> Result { + self.inner.count_tokens(self.prepare_request(request)).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use switchyard_protocol::Metadata; + + fn client(format: WireFormat) -> TargetClient { + TargetClient::new( + "provider/model".into(), + format, + match format { + WireFormat::OpenAiChat => "https://provider.example/v1/chat/completions".into(), + WireFormat::OpenAiResponses => "https://provider.example/v1/responses".into(), + WireFormat::AnthropicMessages => "https://provider.example/v1/messages".into(), + }, + BTreeMap::new(), + ) + .unwrap() + } + + #[test] + fn target_preparation_forces_format_and_removes_inbound_headers() { + let client = client(WireFormat::AnthropicMessages); + let request = Request { + metadata: Some(Metadata { + correlation_id: Some("request-123".into()), + wire_format: Some(WireFormat::OpenAiChat), + http_headers: Some(BTreeMap::from([ + ("authorization".into(), "Bearer caller-secret".into()), + ("x-caller-only".into(), "must-not-forward".into()), + ])), + ..Metadata::default() + }), + ..Request::default() + }; + + let prepared = client.prepare_request(request); + let metadata = prepared.metadata.unwrap(); + assert_eq!(metadata.wire_format, Some(WireFormat::AnthropicMessages)); + assert_eq!(metadata.correlation_id.as_deref(), Some("request-123")); + assert!(metadata.http_headers.is_none()); + } + + #[test] + fn missing_metadata_is_created_for_the_target_format() { + let client = client(WireFormat::OpenAiResponses); + let prepared = client.prepare_request(Request::default()); + assert_eq!( + prepared.metadata.and_then(|metadata| metadata.wire_format), + Some(WireFormat::OpenAiResponses) + ); + } + + #[test] + fn semantic_selection_does_not_replace_the_provider_model() { + let client = client(WireFormat::OpenAiChat); + assert_eq!(client.provider_model(), "provider/model"); + } + + #[test] + fn only_anthropic_targets_advertise_count_tokens() { + assert!(client(WireFormat::AnthropicMessages).supports_count_tokens()); + assert!(!client(WireFormat::OpenAiChat).supports_count_tokens()); + assert!(!client(WireFormat::OpenAiResponses).supports_count_tokens()); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs new file mode 100644 index 000000000..8b28c6abe --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -0,0 +1,770 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use http::Uri; +use http::header::{HeaderName, HeaderValue}; +use serde::Deserialize; +use switchyard_libsy::{ + Algorithm, LlmTarget, LlmTargetSet, LlmTaskClassifier, Random, TaskClassifierConfig, +}; +use switchyard_protocol::{RoutedLlmClient, WireFormat}; + +use crate::client::TargetClient; + +pub(crate) fn protocol_from_call(name: &str) -> Option { + match name { + "openai.chat_completions" => Some(WireFormat::OpenAiChat), + "openai.responses" => Some(WireFormat::OpenAiResponses), + "anthropic.messages" => Some(WireFormat::AnthropicMessages), + _ => None, + } +} + +const fn default_endpoint(protocol: WireFormat) -> &'static str { + match protocol { + WireFormat::OpenAiChat => "/v1/chat/completions", + WireFormat::OpenAiResponses => "/v1/responses", + WireFormat::AnthropicMessages => "/v1/messages", + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetBinding { + model: String, + protocol: WireFormat, + #[serde(default)] + endpoint: String, + base_url: String, + #[serde(default = "default_weight")] + weight: f64, + #[serde(default)] + headers: BTreeMap, + #[serde(default)] + header_env: BTreeMap, +} + +impl TargetBinding { + fn dispatch_url(&self) -> String { + let base = self.base_url.trim_end_matches('/'); + let default = default_endpoint(self.protocol); + if self.endpoint.is_empty() && base.ends_with(default) { + return base.to_string(); + } + let endpoint = if self.endpoint.is_empty() { + default + } else { + &self.endpoint + }; + let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { + &endpoint[3..] + } else { + endpoint + }; + format!("{base}{endpoint}") + } + + fn validate(&self, name: &str) -> Result<(), String> { + if self.model.trim().is_empty() { + return Err(format!("target {name:?} model must be non-empty")); + } + if !self.endpoint.is_empty() && !self.endpoint.starts_with('/') { + return Err(format!( + "target {name:?} endpoint must be empty or begin with '/'" + )); + } + if !self.weight.is_finite() || self.weight < 0.0 { + return Err(format!( + "target {name:?} weight must be finite and nonnegative" + )); + } + validate_dispatch_url(name, self.protocol, &self.dispatch_url())?; + self.validate_headers(name) + } + + fn validate_headers(&self, target_name: &str) -> Result<(), String> { + let mut normalized = BTreeSet::new(); + for (name, value) in &self.headers { + let canonical = validate_header(name, value)?; + if is_sensitive_target_header(&canonical) { + return Err(format!( + "target {target_name:?} header {name:?} must be supplied through header_env so its value is not stored in Relay configuration" + )); + } + if !normalized.insert(canonical) { + return Err(format!( + "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" + )); + } + } + for (name, variable) in &self.header_env { + let canonical = validate_header_name(name)?; + if !normalized.insert(canonical) { + return Err(format!( + "target {target_name:?} configures header {name:?} more than once across headers and header_env" + )); + } + if variable.trim().is_empty() { + return Err(format!( + "environment variable name for target header {name:?} must not be empty" + )); + } + if variable.as_bytes().contains(&b'=') || variable.as_bytes().contains(&b'\0') { + return Err(format!( + "environment variable name for target header {name:?} must not contain '=' or NUL" + )); + } + } + Ok(()) + } + + fn prepare(&self) -> Result { + let mut headers = self.headers.clone(); + for (name, variable) in &self.header_env { + let value = std::env::var(variable) + .map_err(|_| format!("environment variable {variable:?} is not set"))?; + validate_header(name, &value)?; + headers.insert(name.clone(), value); + } + let dispatch_url = self.dispatch_url(); + let client = TargetClient::new(self.model.clone(), self.protocol, dispatch_url, headers) + .map_err(|error| format!("failed to create target HTTP client: {error}"))?; + Ok(PreparedTargetBinding { + client: Arc::new(client), + }) + } +} + +pub(crate) struct PreparedTargetBinding { + pub(crate) client: Arc, +} + +impl PreparedTargetBinding { + fn as_llm_target(&self, semantic_name: &str) -> LlmTarget { + LlmTarget { + semantic_name: semantic_name.to_string(), + llm_client: Some(self.client.clone()), + } + } +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum AlgorithmConfig { + Random { + #[serde(default)] + seed: Option, + }, + LlmClassifier { + classifier_target: String, + weak_target: String, + strong_target: String, + #[serde(default)] + escalation: Option, + #[serde(flatten)] + config: TaskClassifierConfig, + }, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SwitchyardConfig { + version: u32, + #[serde(default)] + pub(crate) priority: i32, + #[serde(default = "default_max_retries")] + max_retries: u32, + algorithm: AlgorithmConfig, + targets: BTreeMap, + default_targets: BTreeMap, +} + +pub(crate) struct PreparedConfig { + pub(crate) max_retries: u32, + pub(crate) algorithm: Arc, + pub(crate) targets: BTreeMap, + pub(crate) default_targets: BTreeMap, +} + +impl SwitchyardConfig { + pub(crate) fn validate(&self) -> Result<(), String> { + self.validate_structure()?; + self.build_algorithm(None).map(drop) + } + + fn validate_structure(&self) -> Result<(), String> { + if self.version != 2 { + return Err(format!( + "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", + self.version + )); + } + if self.max_retries > 10 { + return Err("max_retries must not exceed 10".into()); + } + if self.targets.is_empty() { + return Err("targets must not be empty".into()); + } + if self.default_targets.is_empty() { + return Err("default_targets must not be empty".into()); + } + for (name, target) in &self.targets { + if name.trim().is_empty() { + return Err("target names must be non-empty".into()); + } + target.validate(name)?; + } + for (protocol, fallback) in &self.default_targets { + let target = self + .targets + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; + if target.protocol != *protocol { + return Err(format!( + "default target {fallback:?} must use protocol {}", + protocol.as_str() + )); + } + } + Ok(()) + } + + pub(crate) fn prepare(self) -> Result { + self.validate_structure()?; + let targets = self + .targets + .iter() + .map(|(name, target)| target.prepare().map(|prepared| (name.clone(), prepared))) + .collect::, _>>()?; + let algorithm = self.build_algorithm(Some(&targets))?; + Ok(PreparedConfig { + max_retries: self.max_retries, + algorithm, + targets, + default_targets: self.default_targets, + }) + } + + fn build_algorithm( + &self, + prepared: Option<&BTreeMap>, + ) -> Result, String> { + let target = |name: &str| { + if !self.targets.contains_key(name) { + return Err(format!("algorithm target {name:?} is not configured")); + } + Ok(match prepared { + Some(targets) => targets + .get(name) + .ok_or_else(|| format!("algorithm target {name:?} was not prepared"))? + .as_llm_target(name), + None => LlmTarget { + semantic_name: name.to_string(), + llm_client: None, + }, + }) + }; + + match &self.algorithm { + AlgorithmConfig::Random { seed } => { + let routable = self + .targets + .iter() + .filter(|(_, binding)| binding.weight > 0.0) + .collect::>(); + if routable.is_empty() { + return Err( + "random routing requires at least one positive target weight".into(), + ); + } + let targets = routable + .iter() + .map(|(name, _)| target(name)) + .collect::, _>>()?; + let weights = routable + .iter() + .map(|(_, binding)| binding.weight) + .collect::>(); + Random::new(LlmTargetSet::new(targets), Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::LlmClassifier { + classifier_target, + weak_target, + strong_target, + escalation, + config, + } => { + if escalation.is_some() { + return Err( + "llm_classifier escalation mode is not supported by this plugin version" + .into(), + ); + } + let classifier_binding = self.targets.get(classifier_target).ok_or_else(|| { + format!("algorithm target {classifier_target:?} is not configured") + })?; + if classifier_binding.protocol == WireFormat::AnthropicMessages { + return Err(format!( + "classifier target {classifier_target:?} uses anthropic_messages, which cannot encode the required JSON-schema response format without loss; use an openai_chat or openai_responses target" + )); + } + LlmTaskClassifier::new( + target(classifier_target)?, + target(weak_target)?, + target(strong_target)?, + config.clone(), + ) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + } + } +} + +fn validate_dispatch_url( + target_name: &str, + protocol: WireFormat, + dispatch_url: &str, +) -> Result<(), String> { + let uri = dispatch_url + .parse::() + .map_err(|error| format!("target {target_name:?} has invalid URL: {error}"))?; + if !matches!(uri.scheme_str(), Some("http" | "https")) { + return Err(format!( + "target {target_name:?} base_url must use http or https" + )); + } + let authority = uri + .authority() + .ok_or_else(|| format!("target {target_name:?} URL must include a host"))?; + if authority.host().is_empty() { + return Err(format!("target {target_name:?} URL must include a host")); + } + if authority.as_str().contains('@') { + return Err(format!( + "target {target_name:?} URL must not contain embedded credentials" + )); + } + if uri.query().is_some() { + return Err(format!( + "target {target_name:?} URL query parameters are not supported" + )); + } + + // The current switchyard-llm-client accepts provider base URLs and complete + // canonical endpoints. Reject a custom terminal route here instead of + // allowing Backend::url() to append another provider suffix silently. + let expected_suffix = match protocol { + WireFormat::OpenAiChat => "/chat/completions", + WireFormat::OpenAiResponses => "/responses", + WireFormat::AnthropicMessages => "/v1/messages", + }; + if !uri.path().ends_with(expected_suffix) { + return Err(format!( + "target {target_name:?} endpoint must resolve to a canonical {protocol} route ending in {expected_suffix:?}" + )); + } + Ok(()) +} + +fn validate_header_name(name: &str) -> Result { + let parsed = HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + let canonical = parsed.as_str().to_ascii_lowercase(); + if is_forbidden_target_header(&canonical) { + return Err(format!( + "target header {name:?} is controlled by the HTTP transport and cannot be configured" + )); + } + Ok(canonical) +} + +fn validate_header(name: &str, value: &str) -> Result { + let canonical = validate_header_name(name)?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; + Ok(canonical) +} + +fn is_forbidden_target_header(name: &str) -> bool { + matches!( + name, + "connection" + | "content-length" + | "host" + | "keep-alive" + | "proxy-connection" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) || name.starts_with("x-nemo-relay-internal-") +} + +fn is_sensitive_target_header(name: &str) -> bool { + matches!( + name, + "authorization" + | "cookie" + | "x-api-key" + | "api-key" + | "anthropic-api-key" + | "x-goog-api-key" + ) +} + +const fn default_max_retries() -> u32 { + 3 +} + +const fn default_weight() -> f64 { + 1.0 +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn binding(protocol: WireFormat, model: &str) -> TargetBinding { + TargetBinding { + model: model.into(), + protocol, + endpoint: String::new(), + base_url: "https://provider.example/v1".into(), + weight: 1.0, + headers: BTreeMap::new(), + header_env: BTreeMap::new(), + } + } + + fn config() -> SwitchyardConfig { + SwitchyardConfig { + version: 2, + priority: 0, + max_retries: 3, + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([ + ( + "chat".into(), + binding(WireFormat::OpenAiChat, "provider/chat"), + ), + ( + "responses".into(), + binding(WireFormat::OpenAiResponses, "provider/responses"), + ), + ( + "anthropic".into(), + binding(WireFormat::AnthropicMessages, "provider/anthropic"), + ), + ]), + default_targets: BTreeMap::from([ + (WireFormat::OpenAiChat, "chat".into()), + (WireFormat::OpenAiResponses, "responses".into()), + (WireFormat::AnthropicMessages, "anthropic".into()), + ]), + } + } + + #[test] + fn version_two_random_configuration_builds_clients_without_a_service() { + let config = config(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "random"); + assert_eq!(prepared.targets.len(), 3); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) >= 2) + ); + } + + #[test] + fn version_one_reports_the_service_to_library_migration() { + let mut config = config(); + config.version = 1; + let error = config.validate().unwrap_err(); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("version = 2")); + } + + #[test] + fn target_endpoints_must_be_canonical_for_the_current_http_client() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); + let error = config.validate().unwrap_err(); + assert!(error.contains("ending in \"/chat/completions\"")); + + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); + config.validate().unwrap(); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/custom/chat/completions" + ); + } + + #[test] + fn complete_provider_endpoint_is_not_appended_twice() { + let mut config = config(); + let chat = config.targets.get_mut("chat").unwrap(); + chat.base_url = "https://provider.example/v1/chat/completions/".into(); + assert_eq!( + chat.dispatch_url(), + "https://provider.example/v1/chat/completions" + ); + config.validate().unwrap(); + } + + #[test] + fn absolute_urls_cannot_embed_credentials_or_query_parameters() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().base_url = + "https://user:password@provider.example/v1".into(); + assert!( + config + .validate() + .unwrap_err() + .contains("embedded credentials") + ); + + config.targets.get_mut("chat").unwrap().base_url = + "https://provider.example/v1?api-version=1".into(); + assert!(config.validate().unwrap_err().contains("query parameters")); + } + + #[test] + fn transport_owned_and_case_duplicate_headers_are_rejected() { + let mut host_header_config = config(); + let chat = host_header_config.targets.get_mut("chat").unwrap(); + chat.headers + .insert("Host".into(), "attacker.example".into()); + assert!( + host_header_config + .validate() + .unwrap_err() + .contains("HTTP transport") + ); + + let mut static_secret_config = config(); + static_secret_config + .targets + .get_mut("chat") + .unwrap() + .headers + .insert("Authorization".into(), "Bearer target-secret".into()); + assert!( + static_secret_config + .validate() + .unwrap_err() + .contains("must be supplied through header_env") + ); + + let mut duplicate_config = config(); + let chat = duplicate_config.targets.get_mut("chat").unwrap(); + chat.headers.insert("X-Tenant".into(), "blue".into()); + chat.header_env + .insert("x-tenant".into(), "TARGET_TENANT".into()); + assert!( + duplicate_config + .validate() + .unwrap_err() + .contains("more than once") + ); + } + + #[test] + fn only_canonical_relay_execution_names_resolve_protocols() { + assert_eq!( + protocol_from_call("openai.chat_completions"), + Some(WireFormat::OpenAiChat) + ); + assert_eq!( + protocol_from_call("openai.responses"), + Some(WireFormat::OpenAiResponses) + ); + assert_eq!( + protocol_from_call("anthropic.messages"), + Some(WireFormat::AnthropicMessages) + ); + assert_eq!(protocol_from_call("openai_chat"), None); + } + + #[test] + fn schema_required_contract_fields_do_not_default_during_deserialization() { + let base = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + for field in ["version", "algorithm", "default_targets"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + let error = serde_json::from_value::(value) + .err() + .expect("required field must not default"); + assert!(error.to_string().contains(field), "field={field}: {error}"); + } + } + + #[test] + fn unknown_target_fields_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "unexpected_setting": true + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("unknown target field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); + } + + #[test] + fn unknown_algorithm_fields_are_rejected() { + let error = serde_json::from_value::(json!({ + "kind": "random", + "seed": 42, + "unexpected_setting": true + })) + .err() + .expect("unknown algorithm field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); + } + + #[test] + fn classifier_attaches_clients_to_judge_and_routed_targets() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "recent_turn_window": 4, + "max_output_tokens": 512 + })) + .unwrap(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) >= 2) + ); + } + + #[test] + fn classifier_rejects_anthropic_judge_targets_before_dispatch() { + let mut config = config(); + config.algorithm = AlgorithmConfig::LlmClassifier { + classifier_target: "anthropic".into(), + weak_target: "responses".into(), + strong_target: "chat".into(), + escalation: None, + config: TaskClassifierConfig { + base_threshold: 0.5, + ..Default::default() + }, + }; + + let error = config.validate().unwrap_err(); + assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); + } + + #[test] + fn validation_does_not_resolve_environment_backed_headers() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( + "authorization".into(), + "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), + )]); + + config.validate().unwrap(); + let error = config + .prepare() + .err() + .expect("preparation must resolve headers"); + assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); + } + + #[test] + fn invalid_environment_variable_names_are_rejected_before_resolution() { + for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = + BTreeMap::from([("authorization".into(), variable.into())]); + + let error = config.validate().unwrap_err(); + assert!(error.contains("must not contain '=' or NUL")); + } + } + + #[test] + fn static_validation_preserves_algorithm_constructor_checks() { + let mut random = config(); + for target in random.targets.values_mut() { + target.weight = 0.0; + } + assert!( + random + .validate() + .unwrap_err() + .contains("at least one positive target weight") + ); + + let mut classifier = config(); + classifier.algorithm = AlgorithmConfig::LlmClassifier { + classifier_target: "chat".into(), + weak_target: "responses".into(), + strong_target: "anthropic".into(), + escalation: None, + config: TaskClassifierConfig { + base_threshold: 1.1, + ..Default::default() + }, + }; + assert!( + classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); + } + + #[test] + fn zero_weight_random_targets_are_fallback_only() { + let mut config = config(); + config.targets.get_mut("anthropic").unwrap().weight = 0.0; + let prepared = config.prepare().unwrap(); + + assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); + assert!(Arc::strong_count(&prepared.targets["chat"].client) >= 2); + assert!(Arc::strong_count(&prepared.targets["responses"].client) >= 2); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/executor.rs b/crates/switchyard-nemo-relay-plugin/src/executor.rs new file mode 100644 index 000000000..3dacb5e97 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/executor.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::future::Future; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; + +use tokio::runtime::{Builder, Handle}; +use tokio::sync::oneshot; +use tokio::task::AbortHandle; + +/// Plugin-owned async executor. +/// +/// Relay's generic V3 host table lets the plugin return `Pending`, so neither +/// buffered nor streaming callbacks block Relay runtime workers. Keeping the +/// runtime on a dedicated thread also avoids entering Relay's Tokio runtime +/// from a separately linked cdylib. +#[derive(Clone)] +pub(crate) struct PluginExecutor { + inner: Arc, +} + +struct ExecutorInner { + handle: Handle, + shutdown: Mutex>>, + thread: Mutex>>, +} + +impl PluginExecutor { + pub(crate) fn new() -> Result { + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let thread = thread::Builder::new() + .name("switchyard-relay-http".into()) + .spawn(move || { + let runtime = match Builder::new_multi_thread() + .worker_threads(2) + .thread_name("switchyard-relay-http-worker") + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + if ready_tx + .send(Ok((runtime.handle().clone(), shutdown_tx))) + .is_err() + { + return; + } + runtime.block_on(async { + let _ = shutdown_rx.await; + }); + }) + .map_err(|error| format!("failed to start Switchyard HTTP runtime: {error}"))?; + let (handle, shutdown) = ready_rx + .recv() + .map_err(|_| "Switchyard HTTP runtime stopped during startup".to_string())??; + Ok(Self { + inner: Arc::new(ExecutorInner { + handle, + shutdown: Mutex::new(Some(shutdown)), + thread: Mutex::new(Some(thread)), + }), + }) + } + + pub(crate) fn spawn(&self, future: F) -> AbortHandle + where + F: Future + Send + 'static, + { + self.inner.handle.spawn(future).abort_handle() + } +} + +impl Drop for ExecutorInner { + fn drop(&mut self) { + if let Some(shutdown) = self + .shutdown + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = shutdown.send(()); + } + if let Some(thread) = self + .thread + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + if std::thread::current() + .name() + .is_some_and(|name| name.starts_with("switchyard-relay-http-worker")) + { + // The runtime owner will join this worker after the current + // task returns. Waiting here would deadlock that shutdown. + drop(thread); + } else { + let _ = thread.join(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn executor_runs_buffered_and_spawned_work() { + let executor = PluginExecutor::new().unwrap(); + let (sender, receiver) = mpsc::sync_channel(1); + executor.spawn(async move { + sender.send("done").unwrap(); + }); + assert_eq!(receiver.recv().unwrap(), "done"); + } + + #[test] + fn last_reference_can_drop_on_a_worker() { + let executor = PluginExecutor::new().unwrap(); + let worker_reference = executor.clone(); + let (sender, receiver) = mpsc::sync_channel(1); + executor.spawn(async move { + drop(worker_reference); + sender.send(()).unwrap(); + }); + drop(executor); + receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("dropping the executor on its own worker must not deadlock"); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs new file mode 100644 index 000000000..f5ce094c1 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/ffi.rs @@ -0,0 +1,444 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small ownership wrapper around Relay's generic C host-table v3 hooks. +//! +//! The plugin manifest remains native API v1. Relay 0.7 supplies the appended +//! v3 host table to rebuilt v1 plugins, which lets this crate return `Pending` +//! and settle work from its own runtime without a targeted-continuation ABI. + +use std::ffi::c_void; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use nemo_relay_plugin::{ + Json, LlmRequest, NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, + NemoRelayNativeAsyncNextStreamCb, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, + NemoRelayNativeHostApiV3, NemoRelayNativeScopeHandle, NemoRelayNativeString, NemoRelayStatus, +}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +const BACKPRESSURE_POLL: Duration = Duration::from_millis(1); +const CANCELLATION_POLL: Duration = Duration::from_millis(10); +const MAX_PASSTHROUGH_BUFFER_BYTES: usize = 8 * 1024 * 1024; +const MAX_PASSTHROUGH_BUFFER_EVENTS: usize = 256; + +pub(crate) struct HostString { + host: NemoRelayNativeHostApiV1, + ptr: *mut NemoRelayNativeString, +} + +// Host strings are immutable allocations owned by Relay's thread-safe host table. +unsafe impl Send for HostString {} + +impl HostString { + pub(crate) fn json( + host: &NemoRelayNativeHostApiV1, + value: &impl Serialize, + ) -> Result { + let value = serde_json::to_string(value).map_err(|error| error.to_string())?; + Self::text(host, &value) + } + + pub(crate) fn text(host: &NemoRelayNativeHostApiV1, value: &str) -> Result { + let mut ptr = ptr::null_mut(); + let status = unsafe { (host.string_new)(value.as_ptr(), value.len(), &mut ptr) }; + if status == NemoRelayStatus::Ok && !ptr.is_null() { + Ok(Self { host: *host, ptr }) + } else { + Err(format!("Relay host string allocation failed: {status:?}")) + } + } + + pub(crate) fn as_ptr(&self) -> *const NemoRelayNativeString { + self.ptr + } +} + +impl Drop for HostString { + fn drop(&mut self) { + unsafe { (self.host.string_free)(self.ptr) }; + } +} + +pub(crate) fn read_string( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + if value.is_null() { + return Err("Relay passed a null native string".into()); + } + let len = unsafe { (host.string_len)(value) }; + let data = unsafe { (host.string_data)(value) }; + if data.is_null() && len != 0 { + return Err("Relay passed an invalid native string".into()); + } + let bytes = if len == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(data, len) } + }; + std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|error| error.to_string()) +} + +pub(crate) fn read_json( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + serde_json::from_str(&read_string(host, value)?).map_err(|error| error.to_string()) +} + +/// Captures the current Relay scope as an explicit event parent. +/// +/// Async plugin work runs on a plugin-owned thread, so relying on thread-local +/// scope state would orphan its marks. The host handle is a cloned scope handle +/// and remains valid until this guard is dropped. +pub(crate) struct ParentScope { + host: NemoRelayNativeHostApiV1, + ptr: *mut NemoRelayNativeScopeHandle, +} + +unsafe impl Send for ParentScope {} +unsafe impl Sync for ParentScope {} + +impl ParentScope { + pub(crate) fn capture(host: &NemoRelayNativeHostApiV1) -> Option { + let mut ptr = ptr::null_mut(); + let status = unsafe { (host.scope_get_current)(&mut ptr) }; + (status == NemoRelayStatus::Ok && !ptr.is_null()).then_some(Self { host: *host, ptr }) + } + + pub(crate) fn emit_mark(&self, name: &str, data: &Json, metadata: &Json) -> Result<(), String> { + let name = HostString::text(&self.host, name)?; + let data = HostString::json(&self.host, data)?; + let metadata = HostString::json(&self.host, metadata)?; + let status = unsafe { + (self.host.emit_mark)( + name.as_ptr(), + self.ptr, + data.as_ptr(), + metadata.as_ptr(), + ptr::null(), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "Relay rejected Switchyard routing mark: {status:?}" + )) + } + } +} + +impl Drop for ParentScope { + fn drop(&mut self) { + unsafe { (self.host.scope_handle_free)(self.ptr) }; + } +} + +pub(crate) fn invoke_next_buffered( + host: &NemoRelayNativeHostApiV3, + next: usize, + completion: usize, + request: &LlmRequest, +) -> Result<(), String> { + let request = HostString::json(&host.v1, request)?; + let status = unsafe { + (host.async_next_invoke)( + next as *const NemoRelayNativeAsyncNext, + request.as_ptr(), + completion as *const NemoRelayNativeAsyncCompletion, + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!("Relay rejected buffered pass-through: {status:?}")) + } +} + +enum DownstreamStreamItem { + Chunk { value: Json, encoded_bytes: usize }, +} + +struct DownstreamStreamState { + host: NemoRelayNativeHostApiV1, + sender: mpsc::Sender, + terminal: Option>>, + queued_bytes: Arc, +} + +pub(crate) async fn invoke_next_stream( + host: &NemoRelayNativeHostApiV3, + next: usize, + output: usize, + request: &LlmRequest, +) -> Result<(), String> { + let request = HostString::json(&host.v1, request)?; + let (sender, mut receiver) = mpsc::channel(MAX_PASSTHROUGH_BUFFER_EVENTS); + let (terminal, terminal_result) = oneshot::channel(); + let queued_bytes = Arc::new(AtomicUsize::new(0)); + let state = Box::into_raw(Box::new(DownstreamStreamState { + host: host.v1, + sender, + terminal: Some(terminal), + queued_bytes: Arc::clone(&queued_bytes), + })) + .cast::(); + let status = unsafe { + (host.async_next_invoke_stream)( + next as *const NemoRelayNativeAsyncNext, + request.as_ptr(), + output as *const NemoRelayNativeAsyncStream, + downstream_stream_result as NemoRelayNativeAsyncNextStreamCb, + state, + ) + }; + if status != NemoRelayStatus::Ok { + unsafe { drop(Box::from_raw(state.cast::())) }; + return Err(format!("Relay rejected streaming pass-through: {status:?}")); + } + + while let Some(item) = receiver.recv().await { + match item { + DownstreamStreamItem::Chunk { + value, + encoded_bytes, + } => { + let result = push_stream(host, output, &value).await; + queued_bytes.fetch_sub(encoded_bytes, Ordering::AcqRel); + result?; + } + } + } + terminal_result + .await + .unwrap_or_else(|_| Err("Relay dropped the streaming pass-through callback".into())) +} + +unsafe extern "C" fn downstream_stream_result( + user_data: *mut c_void, + chunk_json: *const NemoRelayNativeString, + error: *const NemoRelayNativeString, + done: bool, +) -> bool { + if !error.is_null() { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + let error = read_string(&state.host, error) + .unwrap_or_else(|_| "Relay streaming pass-through failed".into()); + settle_downstream_stream(state, Err(error)); + return false; + } + if done { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Ok(())); + return false; + } + + let state = unsafe { &*user_data.cast::() }; + let parsed = read_string(&state.host, chunk_json).and_then(|encoded| { + let encoded_bytes = encoded.len(); + let value = serde_json::from_str(&encoded).map_err(|error| error.to_string())?; + Ok((value, encoded_bytes)) + }); + let (value, encoded_bytes) = match parsed { + Ok(parsed) => parsed, + Err(error) => { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Err(error)); + return false; + } + }; + if !reserve_buffer_bytes(&state.queued_bytes, encoded_bytes) { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream( + state, + Err(format!( + "Relay streaming pass-through exceeded its {}-byte queued payload limit", + MAX_PASSTHROUGH_BUFFER_BYTES + )), + ); + return false; + } + + match state.sender.try_send(DownstreamStreamItem::Chunk { + value, + encoded_bytes, + }) { + Ok(()) => true, + Err(error) => { + let (item, message) = match error { + mpsc::error::TrySendError::Full(item) => ( + item, + format!( + "Relay streaming pass-through exceeded its {MAX_PASSTHROUGH_BUFFER_EVENTS}-event queue" + ), + ), + mpsc::error::TrySendError::Closed(item) => ( + item, + "Relay dropped the streaming pass-through receiver".into(), + ), + }; + let encoded_bytes = item.encoded_bytes(); + state + .queued_bytes + .fetch_sub(encoded_bytes, Ordering::AcqRel); + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Err(message)); + false + } + } +} + +impl DownstreamStreamItem { + fn encoded_bytes(&self) -> usize { + match self { + Self::Chunk { encoded_bytes, .. } => *encoded_bytes, + } + } +} + +fn settle_downstream_stream(mut state: Box, result: Result<(), String>) { + if let Some(terminal) = state.terminal.take() { + let _ = terminal.send(result); + } +} + +fn reserve_buffer_bytes(queued: &AtomicUsize, encoded_bytes: usize) -> bool { + queued + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + current + .checked_add(encoded_bytes) + .filter(|next| *next <= MAX_PASSTHROUGH_BUFFER_BYTES) + }) + .is_ok() +} + +pub(crate) async fn wait_for_completion_cancellation( + host: &NemoRelayNativeHostApiV3, + completion: usize, +) { + while !completion_cancelled(host, completion as *const NemoRelayNativeAsyncCompletion) { + tokio::time::sleep(CANCELLATION_POLL).await; + } +} + +pub(crate) async fn wait_for_stream_cancellation(host: &NemoRelayNativeHostApiV3, stream: usize) { + while !unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + tokio::time::sleep(CANCELLATION_POLL).await; + } +} + +pub(crate) fn completion_cancelled( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, +) -> bool { + unsafe { (host.async_completion_is_cancelled)(completion) } +} + +pub(crate) fn resolve_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, + value: &Json, +) -> NemoRelayStatus { + match HostString::json(&host.v1, value) { + Ok(value) => unsafe { (host.async_completion_resolve_json)(completion, value.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub(crate) fn reject_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, + message: &str, +) -> NemoRelayStatus { + match HostString::text(&host.v1, message) { + Ok(message) => unsafe { (host.async_completion_reject)(completion, message.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub(crate) async fn push_stream( + host: &NemoRelayNativeHostApiV3, + stream: usize, + value: &Json, +) -> Result<(), String> { + let value = HostString::json(&host.v1, value)?; + loop { + if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + return Err("Relay caller cancelled the output stream".into()); + } + match unsafe { + (host.async_stream_push_json)( + stream as *const NemoRelayNativeAsyncStream, + value.as_ptr(), + ) + } { + NemoRelayStatus::Ok => return Ok(()), + // Native API v1 reports its bounded queue's WouldBlock state as Internal. + NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, + status => return Err(format!("Relay rejected output stream event: {status:?}")), + } + } +} + +pub(crate) fn finish_stream( + host: &NemoRelayNativeHostApiV3, + stream: *const NemoRelayNativeAsyncStream, +) -> NemoRelayStatus { + unsafe { (host.async_stream_finish)(stream) } +} + +pub(crate) async fn reject_stream( + host: &NemoRelayNativeHostApiV3, + stream: usize, + message: &str, +) -> NemoRelayStatus { + let Ok(message) = HostString::text(&host.v1, message) else { + return NemoRelayStatus::Internal; + }; + loop { + if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + return NemoRelayStatus::InvalidArg; + } + match unsafe { + (host.async_stream_reject)( + stream as *const NemoRelayNativeAsyncStream, + message.as_ptr(), + ) + } { + NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, + status => return status, + } + } +} + +pub(crate) unsafe fn release_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, +) { + unsafe { (host.async_completion_release)(completion) }; +} + +pub(crate) unsafe fn release_next( + host: &NemoRelayNativeHostApiV3, + next: *const NemoRelayNativeAsyncNext, +) { + unsafe { (host.async_next_release)(next) }; +} + +pub(crate) unsafe fn release_stream( + host: &NemoRelayNativeHostApiV3, + stream: *const NemoRelayNativeAsyncStream, +) { + unsafe { (host.async_stream_release)(stream) }; +} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs new file mode 100644 index 000000000..8f6ba68b6 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -0,0 +1,401 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod client; +mod config; +mod executor; +mod ffi; +mod runtime; +mod translation; + +use std::ffi::c_void; +use std::mem; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; + +use futures_util::FutureExt; +use nemo_relay_plugin::{ + ConfigDiagnostic, DiagnosticLevel, Json, NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE, + NativePlugin, NemoRelayNativeAsyncCallbackState, NemoRelayNativeAsyncCompletion, + NemoRelayNativeAsyncMiddlewareKind, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, + NemoRelayNativeHostApiV3, NemoRelayNativeString, NemoRelayStatus, PluginContext, +}; +use serde::Deserialize; +use serde_json::Map; + +use crate::config::SwitchyardConfig; +use crate::executor::PluginExecutor; +use crate::runtime::SwitchyardRuntime; + +#[derive(Deserialize)] +struct Invocation { + name: String, + request: nemo_relay_plugin::LlmRequest, +} + +struct CallbackState { + host: NemoRelayNativeHostApiV3, + runtime: Arc, + executor: PluginExecutor, +} + +#[derive(Default)] +struct SwitchyardPlugin; + +impl NativePlugin for SwitchyardPlugin { + fn plugin_kind(&self) -> &str { + "nvidia.switchyard" + } + + fn allows_multiple_components(&self) -> bool { + false + } + + fn validate(&self, plugin_config: &Map) -> Vec { + match parse_config(plugin_config).and_then(|config| config.validate()) { + Ok(()) => Vec::new(), + Err(message) => vec![ConfigDiagnostic { + level: DiagnosticLevel::Error, + code: "switchyard.invalid_config".into(), + component: Some("nvidia.switchyard".into()), + field: Some("config".into()), + message, + }], + } + } + + fn register( + &mut self, + plugin_config: &Map, + ctx: &mut PluginContext<'_>, + ) -> nemo_relay_plugin::Result<()> { + let host_v1 = ctx.host_api(); + if host_v1.abi_version < NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE + || host_v1.struct_size < mem::size_of::() + { + return Err( + "Switchyard requires Relay 0.7 or newer with the generic asynchronous native host table" + .into(), + ); + } + let host = unsafe { *(host_v1 as *const _ as *const NemoRelayNativeHostApiV3) }; + let config = parse_config(plugin_config)?; + let priority = config.priority; + let state = Arc::new(CallbackState { + host, + runtime: Arc::new(SwitchyardRuntime::new(config)?), + executor: PluginExecutor::new()?, + }); + + register_buffered(ctx, priority, Arc::clone(&state))?; + register_stream(ctx, priority, state)?; + Ok(()) + } +} + +fn register_buffered( + ctx: &mut PluginContext<'_>, + priority: i32, + state: Arc, +) -> Result<(), String> { + let user_data = Box::into_raw(Box::new(state)).cast::(); + let status = unsafe { + ctx.register_async_middleware_raw( + NemoRelayNativeAsyncMiddlewareKind::LlmExecutionIntercept, + "switchyard.run_stream.buffered", + priority, + false, + buffered_callback, + user_data, + Some(free_callback_state), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "failed to register Switchyard buffered execution: {status:?}" + )) + } +} + +fn register_stream( + ctx: &mut PluginContext<'_>, + priority: i32, + state: Arc, +) -> Result<(), String> { + let user_data = Box::into_raw(Box::new(state)).cast::(); + let status = unsafe { + ctx.register_async_stream_middleware_raw( + "switchyard.run_stream.streaming", + priority, + stream_callback, + user_data, + Some(free_callback_state), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "failed to register Switchyard streaming execution: {status:?}" + )) + } +} + +fn parse_config(plugin_config: &Map) -> Result { + match plugin_config.get("version").and_then(Json::as_u64) { + Some(2) => {} + Some(version) => { + return Err(format!( + "unsupported Switchyard config version {version}; version 1 used switchyard-server; migrate to version = 2" + )); + } + None => { + return Err("invalid Switchyard configuration: version must be the integer 2".into()); + } + } + serde_json::from_value(Json::Object(plugin_config.clone())) + .map_err(|error| format!("invalid Switchyard configuration: {error}")) +} + +unsafe extern "C" fn free_callback_state(user_data: *mut c_void) { + if !user_data.is_null() { + unsafe { drop(Box::from_raw(user_data.cast::>())) }; + } +} + +unsafe extern "C" fn buffered_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + completion: *const NemoRelayNativeAsyncCompletion, +) -> u32 { + if user_data.is_null() || completion.is_null() || next.is_null() { + return NemoRelayNativeAsyncCallbackState::Complete as u32; + } + let state = unsafe { &*user_data.cast::>() }.clone(); + let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|error| error.to_string()) + }); + let next = next as usize; + let completion = completion as usize; + let invocation = match invocation { + Ok(invocation) => invocation, + Err(error) => { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &format!("invalid Relay LLM invocation: {error}"), + ); + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + } + }; + let Some(inbound) = state.runtime.managed_protocol(&invocation.name) else { + if let Err(error) = + ffi::invoke_next_buffered(&state.host, next, completion, &invocation.request) + { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &error, + ); + } + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + }; + let request = match state + .runtime + .decode_request(inbound, &invocation.request, false) + { + Ok(request) => request, + Err(error) => { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &error, + ); + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + } + }; + let parent = ffi::ParentScope::capture(&state.host.v1); + let task_state = Arc::clone(&state); + state.executor.spawn(async move { + let execution = AssertUnwindSafe(task_state.runtime.execute_buffered( + inbound, + request, + parent.as_ref(), + )) + .catch_unwind(); + tokio::pin!(execution); + let result = tokio::select! { + biased; + () = ffi::wait_for_completion_cancellation(&task_state.host, completion) => None, + result = &mut execution => Some( + result.unwrap_or_else(|_| Err("Switchyard buffered execution panicked".into())) + ), + }; + + let completion_ptr = completion as *const NemoRelayNativeAsyncCompletion; + if let Some(result) = result { + match result { + Ok(response) => { + let _ = ffi::resolve_completion(&task_state.host, completion_ptr, &response); + } + Err(error) => { + let _ = ffi::reject_completion(&task_state.host, completion_ptr, &error); + } + } + } + unsafe { + ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion(&task_state.host, completion_ptr); + } + }); + NemoRelayNativeAsyncCallbackState::Pending as u32 +} + +unsafe extern "C" fn stream_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, +) -> u32 { + if user_data.is_null() || output.is_null() || next.is_null() { + return NemoRelayNativeAsyncCallbackState::Complete as u32; + } + let state = unsafe { &*user_data.cast::>() }.clone(); + let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|error| error.to_string()) + }); + let managed_protocol = invocation + .as_ref() + .ok() + .and_then(|invocation| state.runtime.managed_protocol(&invocation.name)); + let parent = managed_protocol.and_then(|_| ffi::ParentScope::capture(&state.host.v1)); + let next = next as usize; + let output = output as usize; + let task_state = Arc::clone(&state); + state.executor.spawn(async move { + let execution = AssertUnwindSafe(async { + match invocation { + Ok(invocation) => { + if let Some(inbound) = managed_protocol { + match task_state + .runtime + .decode_request(inbound, &invocation.request, true) + { + Ok(request) => { + task_state + .runtime + .execute_stream( + &task_state.host, + output, + inbound, + request, + parent.as_ref(), + ) + .await + } + Err(error) => Err(error), + } + } else { + ffi::invoke_next_stream(&task_state.host, next, output, &invocation.request) + .await + } + } + Err(error) => Err(format!("invalid Relay LLM stream invocation: {error}")), + } + }) + .catch_unwind(); + tokio::pin!(execution); + let result = tokio::select! { + biased; + () = ffi::wait_for_stream_cancellation(&task_state.host, output) => None, + result = &mut execution => Some( + result.unwrap_or_else(|_| Err("Switchyard streaming execution panicked".into())) + ), + }; + + match result { + Some(Ok(())) => { + let _ = ffi::finish_stream( + &task_state.host, + output as *const NemoRelayNativeAsyncStream, + ); + } + Some(Err(error)) => { + let _ = ffi::reject_stream(&task_state.host, output, &error).await; + } + None => {} + } + unsafe { + ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_stream( + &task_state.host, + output as *const NemoRelayNativeAsyncStream, + ); + } + }); + NemoRelayNativeAsyncCallbackState::Pending as u32 +} + +nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn version_one_service_config_gets_a_migration_error_before_v2_deserialization() { + let value = json!({ + "version": 1, + "service_url": "http://127.0.0.1:8080", + "health_endpoint": "/healthz" + }); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("version one must be rejected"); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("migrate to version = 2")); + assert!(!error.contains("unknown field")); + } + + #[test] + fn version_must_be_an_integer() { + let value = json!({"version": "2"}); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("non-integer versions must be rejected"); + assert_eq!( + error, + "invalid Switchyard configuration: version must be the integer 2" + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs new file mode 100644 index 000000000..f9cfea69f --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -0,0 +1,601 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::sync::Arc; + +use futures_util::{StreamExt, stream}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, NemoRelayNativeHostApiV3}; +use serde_json::{Map, json}; +use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; +use switchyard_protocol::{ + Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, SimpleDecision, + WireFormat, +}; +use switchyard_translation::{TranslationEngine, encode_stream}; + +use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; +use crate::ffi::ParentScope; +use crate::{ffi, translation}; + +pub(crate) struct SwitchyardRuntime { + max_retries: u32, + algorithm: Arc, + targets: BTreeMap, + default_targets: BTreeMap, + translation: TranslationEngine, +} + +impl SwitchyardRuntime { + pub(crate) fn new(config: SwitchyardConfig) -> Result { + let prepared = config.prepare()?; + Ok(Self { + max_retries: prepared.max_retries, + algorithm: prepared.algorithm, + targets: prepared.targets, + default_targets: prepared.default_targets, + translation: TranslationEngine::default(), + }) + } + + pub(crate) fn managed_protocol(&self, name: &str) -> Option { + protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) + } + + pub(crate) fn decode_request( + &self, + inbound: WireFormat, + request: &RelayRequest, + streaming: bool, + ) -> Result { + let mut llm_request = translation::decode_request(&self.translation, inbound, request)?; + llm_request.stream = streaming; + let headers = string_headers(&request.headers); + let mut metadata = Metadata::from_headers(&headers); + // Keep identity/routing metadata, but target clients deliberately clear + // these caller headers before HTTP dispatch. + metadata.http_headers = Some(headers); + metadata.wire_format = Some(inbound); + Ok(Request { + llm_request, + raw_request: Some(request.content.clone()), + metadata: Some(metadata), + }) + } + + pub(crate) async fn execute_buffered( + &self, + inbound: WireFormat, + request: Request, + parent: Option<&ParentScope>, + ) -> Result { + let metadata = identity_metadata(request.metadata.as_ref()); + let max_attempts = self.max_retries + 1; + let mut attempt = 1; + loop { + self.mark( + parent, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + &metadata, + ); + match self + .drive(request.clone(), attempt, parent, &metadata) + .await + { + Ok(response) => { + let LlmResponse::Agg(response) = response.llm_response else { + return Err("libsy returned a stream for a buffered request".into()); + }; + return translation::encode_response(&self.translation, inbound, &response); + } + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { + self.mark( + parent, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + attempt += 1; + } + Err(failure) => { + self.mark( + parent, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + return self + .fallback_buffered(inbound, request, parent, &metadata) + .await; + } + } + } + } + + pub(crate) async fn execute_stream( + &self, + host: &NemoRelayNativeHostApiV3, + output: usize, + inbound: WireFormat, + request: Request, + parent: Option<&ParentScope>, + ) -> Result<(), String> { + let metadata = identity_metadata(request.metadata.as_ref()); + let max_attempts = self.max_retries + 1; + let mut attempt = 1; + loop { + self.mark( + parent, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + &metadata, + ); + let (response, fallback_used) = match self + .drive(request.clone(), attempt, parent, &metadata) + .await + { + Ok(response) => (response, false), + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { + self.mark( + parent, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + attempt += 1; + continue; + } + Err(failure) => { + self.mark( + parent, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + ( + self.fallback_response(inbound, request.clone(), parent, &metadata) + .await?, + true, + ) + } + }; + + let mut events = match returned_events(response, inbound).await { + Ok(events) => events, + Err(failure) + if !fallback_used + && libsy_error_retryable(&failure) + && attempt < max_attempts => + { + self.mark( + parent, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + attempt += 1; + continue; + } + Err(failure) if !fallback_used => { + self.mark( + parent, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), parent, &metadata) + .await?; + returned_events(fallback, inbound) + .await + .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? + } + Err(failure) => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + }; + + let mut committed = false; + while let Some(item) = events.next().await { + match item { + Ok(event) => { + ffi::push_stream(host, output, &event).await?; + committed = true; + } + Err(failure) + if !fallback_used + && !committed + && libsy_error_retryable(&failure) + && attempt < max_attempts => + { + self.mark( + parent, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + attempt += 1; + break; + } + Err(failure) if !fallback_used && !committed => { + self.mark( + parent, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), parent, &metadata) + .await?; + let mut fallback = + returned_events(fallback, inbound).await.map_err(|error| { + public_libsy_failure("trusted fallback stream", &error) + })?; + while let Some(item) = fallback.next().await { + let event = item.map_err(|error| { + public_libsy_failure("trusted fallback stream", &error) + })?; + ffi::push_stream(host, output, &event).await?; + } + return Ok(()); + } + Err(failure) if !committed => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + Err(failure) => { + self.mark( + parent, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + return Err(public_libsy_failure( + "Switchyard stream failed after response commitment", + &failure, + )); + } + } + } + if committed { + return Ok(()); + } + // A retry path breaks the event loop before commitment and starts a + // fresh libsy run. A successfully encoded stream always emits at + // least one event because `returned_events` rejects empty inputs. + } + } + + async fn drive( + &self, + request: Request, + attempt: u32, + parent: Option<&ParentScope>, + mark_metadata: &Json, + ) -> Result { + let context = context_from_metadata(request.metadata.as_ref()); + let mut steps = self.algorithm.clone().run_stream(context, request, None); + while let Some(step) = steps.next().await { + match step { + Ok(Step::Decision(decision)) => { + self.emit_decision(parent, decision.as_ref(), attempt, mark_metadata); + } + Ok(Step::CallLlm(call)) => self.serve_call(*call).await?, + Ok(Step::ReturnToAgent(response)) => return Ok(*response), + Err(error) => return Err(error), + } + } + Err(LibsyError::MissingFinalResponse) + } + + async fn serve_call(&self, call: CallLlmRequest) -> switchyard_libsy::Result<()> { + let routed = call.get_routed().clone(); + let target_name = routed.decision.selected_model().to_string(); + let result = match routed.default_client { + Some(client) => client + .call(routed.ctx, routed.request, routed.decision) + .await + .map_err(|source| LibsyError::client_call(target_name, source)), + None => Err(LibsyError::client_call( + target_name, + LlmClientError::Configuration { + message: "libsy CallLlm step has no Switchyard HTTP client".into(), + }, + )), + }; + call.respond(result) + } + + async fn fallback_buffered( + &self, + inbound: WireFormat, + request: Request, + parent: Option<&ParentScope>, + metadata: &Json, + ) -> Result { + let response = self + .fallback_response(inbound, request, parent, metadata) + .await?; + let LlmResponse::Agg(response) = response.llm_response else { + return Err("trusted fallback returned a stream for a buffered request".into()); + }; + translation::encode_response(&self.translation, inbound, &response) + } + + async fn fallback_response( + &self, + inbound: WireFormat, + request: Request, + parent: Option<&ParentScope>, + metadata: &Json, + ) -> Result { + let target_name = self.default_target(inbound)?; + let target = self.target(target_name)?; + self.mark( + parent, + "switchyard.routing.fallback", + json!({"selected_target": target_name}), + metadata, + ); + let decision: Arc = Arc::new(SimpleDecision { + selected_model: target_name.to_string(), + reasoning: Some("trusted fallback target".into()), + }); + let context = context_from_metadata(request.metadata.as_ref()); + target + .client + .call(context, request, decision) + .await + .map_err(|error| public_client_failure("trusted fallback", &error)) + } + + fn target(&self, name: &str) -> Result<&PreparedTargetBinding, String> { + self.targets + .get(name) + .ok_or_else(|| format!("libsy selected unknown target {name:?}")) + } + + fn default_target(&self, protocol: WireFormat) -> Result<&str, String> { + self.default_targets + .get(&protocol) + .map(String::as_str) + .ok_or_else(|| format!("managed protocol {protocol} has no default target")) + } + + fn mark(&self, parent: Option<&ParentScope>, name: &str, data: Json, metadata: &Json) { + let Some(parent) = parent else { return }; + if let Err(error) = parent.emit_mark(name, &data, metadata) { + eprintln!("Switchyard could not emit routing mark {name:?}: {error}"); + } + } + + fn emit_decision( + &self, + parent: Option<&ParentScope>, + decision: &dyn Decision, + attempt: u32, + metadata: &Json, + ) { + self.mark( + parent, + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": decision.selected_model(), + "reasoning": decision.reasoning(), + "routing_tier": decision.routing_tier(), + "is_routed_call": decision.is_routed_call(), + }), + metadata, + ); + } +} + +type ReturnedEventStream = + std::pin::Pin> + Send>>; + +async fn returned_events( + response: Response, + inbound: WireFormat, +) -> Result { + let chunks = match response.llm_response { + LlmResponse::Agg(response) => response.into_stream(), + LlmResponse::Stream(mut chunks) => { + let Some(first) = chunks.next().await else { + return Err(LibsyError::client_call( + "return_to_agent", + LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }, + )); + }; + Box::pin(stream::once(async move { first }).chain(chunks)) + } + }; + let events = encode_stream(chunks, inbound, None) + .map_err(|error| LibsyError::client_call("return_to_agent", error))?; + Ok(Box::pin(events.map(|item| { + item.map_err(|source| match source.downcast::() { + Ok(source) => LibsyError::client_call("return_to_agent", *source), + Err(source) => LibsyError::client_call( + "return_to_agent", + LlmClientError::ResponseTranslation(source.to_string()), + ), + }) + }))) +} + +fn libsy_error_retryable(error: &LibsyError) -> bool { + let LibsyError::ClientCall { source, .. } = error else { + return false; + }; + match source { + LlmClientError::UpstreamHttp { status, .. } => { + matches!(*status, 408 | 425 | 429 | 500 | 502 | 503 | 504) + } + LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true, + _ => false, + } +} + +fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { + let mut data = Map::from_iter([ + ("attempt".into(), Json::from(attempt)), + ( + "retryable".into(), + Json::from(libsy_error_retryable(failure)), + ), + ]); + match failure { + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } => { + data.insert("failure_kind".into(), Json::from("http")); + data.insert("http_status".into(), Json::from(*status)); + } + LibsyError::ClientCall { source, .. } => { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert( + "non_http_kind".into(), + Json::from(client_error_label(source)), + ); + } + _ => { + data.insert("failure_kind".into(), Json::from("algorithm")); + } + } + Json::Object(data) +} + +fn client_error_label(error: &LlmClientError) -> &'static str { + match error { + LlmClientError::InvalidRequest { .. } => "invalid_request", + LlmClientError::RequestTranslation(_) => "request_translation", + LlmClientError::RequestEncoding(_) => "request_encoding", + LlmClientError::ResponseTranslation(_) => "response_translation", + LlmClientError::Configuration { .. } => "configuration", + LlmClientError::Transport { .. } => "transport", + LlmClientError::Timeout { .. } => "timeout", + LlmClientError::ContextWindowExceeded { .. } => "context_window_exceeded", + LlmClientError::UpstreamHttp { .. } => "http", + LlmClientError::InvalidResponse { .. } => "invalid_response", + LlmClientError::Ffi { .. } => "ffi", + LlmClientError::General(_) => "general", + _ => "unknown", + } +} + +fn public_libsy_failure(prefix: &str, error: &LibsyError) -> String { + match error { + LibsyError::ClientCall { source, .. } => public_client_failure(prefix, source), + _ => format!("{prefix}: Switchyard algorithm failure"), + } +} + +fn public_client_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::UpstreamHttp { status, .. } => { + format!("{prefix}: provider returned HTTP {status}") + } + _ => format!("{prefix}: provider {} failure", client_error_label(error)), + } +} + +fn string_headers(headers: &Map) -> BTreeMap { + headers + .iter() + .filter_map(|(name, value)| value.as_str().map(|value| (name.clone(), value.into()))) + .collect() +} + +fn identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()), + "task_id": metadata.and_then(|value| value.task_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) +} + +fn context_from_metadata(metadata: Option<&Metadata>) -> Context { + let Some(metadata) = metadata else { + return Context::default(); + }; + let mut values = std::collections::HashMap::new(); + for (name, value) in [ + ("session_id", metadata.session_id.as_deref()), + ("agent_id", metadata.agent_id.as_deref()), + ("parent_agent_id", metadata.parent_agent_id.as_deref()), + ("agent_kind", metadata.agent_kind.as_deref()), + ("agent_role", metadata.agent_role.as_deref()), + ("task_id", metadata.task_id.as_deref()), + ("task_kind", metadata.task_kind.as_deref()), + ("turn_id", metadata.turn_id.as_deref()), + ("correlation_id", metadata.correlation_id.as_deref()), + ] { + if let Some(value) = value { + values.insert(name.to_string(), value.to_string()); + } + } + values.insert("is_subagent".into(), metadata.is_subagent.to_string()); + values.insert( + "is_delegated_work".into(), + metadata.is_delegated_work.to_string(), + ); + if let Some(session_final) = metadata.session_final { + values.insert("session_final".into(), session_final.to_string()); + } + if let Some(extra) = &metadata.extra_metadata { + for (name, value) in extra { + values.entry(name.clone()).or_insert_with(|| value.clone()); + } + } + let mut context = Context::default(); + context.values = values; + context +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn context_carries_identity_without_http_headers() { + let context = context_from_metadata(Some(&Metadata { + session_id: Some("session-1".into()), + agent_id: Some("agent-1".into()), + is_subagent: true, + extra_metadata: Some(BTreeMap::from([("tenant".into(), "blue".into())])), + http_headers: Some(BTreeMap::from([( + "authorization".into(), + "Bearer caller-secret".into(), + )])), + ..Metadata::default() + })); + + assert_eq!( + context.values.get("session_id").map(String::as_str), + Some("session-1") + ); + assert_eq!( + context.values.get("agent_id").map(String::as_str), + Some("agent-1") + ); + assert_eq!( + context.values.get("is_subagent").map(String::as_str), + Some("true") + ); + assert_eq!( + context.values.get("tenant").map(String::as_str), + Some("blue") + ); + assert!(!context.values.contains_key("authorization")); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs new file mode 100644 index 000000000..6f6f965fe --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay_plugin::LlmRequest as RelayRequest; +use serde_json::Value as Json; +use switchyard_protocol::{AggLlmResponse, LlmRequest, WireFormat}; +use switchyard_translation::{ + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, +}; + +pub(crate) fn decode_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &RelayRequest, +) -> Result { + let output = engine + .decode_request(protocol, &request.content, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.request) +} + +pub(crate) fn validate_target_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &LlmRequest, +) -> Result<(), String> { + let output = engine + .encode_request(protocol, request, &request_policy(protocol)) + .map_err(error)?; + safe(&output.diagnostics) +} + +pub(crate) fn encode_response( + engine: &TranslationEngine, + protocol: WireFormat, + response: &AggLlmResponse, +) -> Result { + let output = engine + .encode_response(protocol, response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.body) +} + +fn policy() -> TranslationPolicy { + TranslationPolicy { + unknown_field_policy: UnknownFieldPolicy::Preserve, + lossy_conversion_policy: LossyConversionPolicy::Reject, + deterministic_ids: DeterministicIdPolicy::GenerateStable { + prefix: "relay".into(), + }, + preservation: PreservationPolicy::InMemory, + target_capabilities: TargetCapabilities::default(), + } +} + +fn request_policy(protocol: WireFormat) -> TranslationPolicy { + let mut policy = policy(); + if protocol == WireFormat::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + +fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { + let unsafe_diagnostics = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if unsafe_diagnostics.is_empty() { + Ok(()) + } else { + Err(format!( + "Switchyard translation was not lossless: {unsafe_diagnostics:?}" + )) + } +} + +fn error(error: switchyard_translation::TranslationError) -> String { + format!("Switchyard translation failed: {error}") +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, json}; + + use super::*; + + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let request = RelayRequest { + headers: Map::new(), + content: json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + let engine = TranslationEngine::default(); + let decoded = decode_request(&engine, WireFormat::OpenAiChat, &request).unwrap(); + validate_target_request(&engine, WireFormat::OpenAiChat, &decoded).unwrap(); + assert_eq!( + decoded + .preservation + .requests + .get(&WireFormat::OpenAiChat.into()) + .and_then(|body| body.get("provider_extension")), + Some(&json!({"exact": true})) + ); + } +} From 8403391b37f31c91c34f1e0e878503095e732ee0 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 16:47:19 -0600 Subject: [PATCH 02/15] docs(relay): document native plugin integration Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 16 ++ README.md | 2 + crates/switchyard-nemo-relay-plugin/README.md | 250 ++++++++++++++++++ docs/index.md | 2 + 4 files changed, 270 insertions(+) create mode 100644 crates/switchyard-nemo-relay-plugin/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 63195a865..2cadfdbc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **NeMo Relay native plugin** — a dynamically loaded integration that runs + libsy's weighted-random and LLM-classifier algorithms in process while + Switchyard owns provider HTTP dispatch, credentials, translation, retries, + and fallback. Managed calls require NeMo Relay 0.7 or newer and do not depend + on `switchyard-server`. + +### Changed + +- **Switchyard HTTP transport limits** — provider redirects are rejected, + connection and read-inactivity timeouts are enforced, buffered success and + error bodies are bounded, and oversized SSE events fail before unbounded line + buffering. Provider error bodies remain available in typed errors but are no + longer included in their default display text. + ### Removed - **Latency-aware router** — the `latency_service` route type and its diff --git a/README.md b/README.md index 751ad11c6..1a420c801 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ algorithm you write yourself. - **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats - **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm - **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead +- **NeMo Relay Plugin**: run random or LLM-classifier routing in Relay while Switchyard owns provider HTTP dispatch ## Quick Start @@ -119,6 +120,7 @@ configured LLM client selects one upstream format. - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation +- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin ## Community diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md new file mode 100644 index 000000000..c0b984088 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -0,0 +1,250 @@ + + +# Switchyard NeMo Relay Dynamic Plugin + +This crate builds the external `nvidia.switchyard` native plugin. It embeds +`switchyard-libsy`, drives `Algorithm::run_stream`, and uses +`switchyard-llm-client` for provider HTTP calls. Managed calls use Relay's +generic asynchronous middleware hooks and do not require a targeted provider +continuation from Relay. + +The plugin uses NeMo Relay native API v1. It depends on the small +`nemo-relay-plugin` authoring SDK, not the Relay runtime, and does not start +`switchyard-server`. Managed provider calls do not use Relay's provider +continuation. + +## Ownership boundary + +For a managed LLM call: + +1. Relay invokes the native LLM execution intercept. +2. The plugin decodes the caller JSON through `switchyard-translation`. +3. The plugin drives the configured libsy algorithm with `run_stream` and + records each genuine decision. +4. For every `CallLlm`, the plugin's `switchyard-llm-client` instance translates + the neutral request, applies the selected target's URL and credentials, and + performs the HTTP request. +5. The plugin passes the real buffered response or response stream to + `CallLlmRequest::respond` and continues until `ReturnToAgent`. +6. The plugin encodes the final neutral response into the caller's protocol. + +Relay still owns the outer LLM lifecycle, dynamic-plugin loading, plugin +configuration, and event substrate. Relay's downstream LLM continuation is +used only for calls whose inbound protocol is not managed by this plugin. + +```mermaid +flowchart LR + A["Caller JSON"] --> B["Relay LLM execution intercept"] + B --> C["Switchyard decode"] + C --> D["libsy run_stream"] + D --> E["CallLlm"] + E --> F["switchyard-llm-client"] + F --> G["Provider HTTP endpoint"] + G --> H["Switchyard response or event decode"] + H --> I["CallLlmRequest.respond"] + I --> J["ReturnToAgent"] + J --> K["Switchyard encode"] + K --> A + + U["Unmanaged profile"] -.-> V["Relay v1 continuation"] +``` + +This boundary has two important consequences: + +- Managed provider calls do not traverse Relay middleware registered after the + Switchyard intercept and do not use the host's provider callback. Provider + transport activity is therefore not represented as nested Relay LLM + lifecycle events. Relay records the outer managed call and the plugin emits + Switchyard routing marks; bridging Switchyard transport spans into Relay is + future work. With an exported Agent or turn scope, Relay 0.7 projects the + outer LLM span and routing-decision span as siblings in one OpenInference + trace. The native callback cannot parent an asynchronous decision mark + directly under the active LLM event; an embedded host that invokes an LLM + without an exported parent scope would therefore produce an orphan decision + span. +- Switchyard owns provider URLs, credentials, HTTP retry behavior, and + translation for managed calls. Relay neither validates nor transports those + target details. + +## Native API v1 and asynchronous execution + +The manifest remains `compat.native_api = "1"`, but this rebuilt plugin requires +the generic C host-table v3 extension shipped by Relay 0.7. It registers through +v3's completion-based buffered and incremental streaming hooks, returns +`Pending` immediately, and performs libsy and HTTP work on a plugin-owned Tokio +runtime. Relay's runtime workers therefore do not wait synchronously for +provider I/O. + +The v3 stream hook retains Relay's bounded 32-event output queue. The plugin +retries a logical event when that queue is full, checks cancellation between +attempts, and releases every completion, continuation, stream, and captured +scope handle exactly once. Managed HTTP work is selected against Relay caller +cancellation, so dropping a buffered or streaming call drops its in-flight +request future. + +Unmanaged profiles use the same generic v3 continuation hooks for pass-through. +V3's downstream stream callback has continue/cancel control but no asynchronous +acknowledgement, so it cannot provide true end-to-end backpressure. The adapter +uses a nonblocking bridge capped at 8 MiB of queued, encoded event payloads; it +also caps the queue at 256 events and safely rejects a pass-through stream that +outruns either bound. The byte cap does not include transient JSON parsing or +in-memory representation overhead. Direct host forwarding without this bound would +still require a small generic Relay hook, but not a targeted-provider contract. +Managed Switchyard streams do not use this pass-through bridge. None of the +managed HTTP or routing operations requires a targeted LLM continuation. + +This is still a raw C boundary: Switchyard contains a small ownership adapter +for host strings and v3 handles because Relay 0.7 does not expose a safe Rust +facade for the generic async surface. That adapter is transport-independent; +all HTTP and routing behavior remains in Switchyard. + +## Supported routers + +This initial plugin supports exactly two libsy algorithms: + +- seeded, weighted `random` routing; and +- capability-based `llm_classifier` routing, where a judge selects the weak or + strong target before the final provider call. + +`stage_router` and response-judging escalation are intentionally deferred. +Unsupported algorithm kinds are rejected instead of being approximated. + +The plugin owns the outer routing retry loop. Each retry starts a fresh libsy +run. Random routing draws again; an algorithm configured with persistent state, +such as classifier session affinity, may intentionally retain its assignment. +Each target's built-in HTTP retry count is set to zero to avoid retrying a +failed target invisibly before reselection. A random target with `weight = 0` +is fallback-only and is not considered by the algorithm. Trusted fallback is +attempted at most once and, for streaming responses, only before the first +caller event is emitted. Outer routing retries are immediate and do not honor +provider `Retry-After` headers. + +## Translation and stream fidelity + +`switchyard-translation` is the only request, response, and event translation +layer. It decodes caller JSON into Switchyard's neutral protocol, encodes each +selected call for the target protocol, decodes provider results, and encodes +`ReturnToAgent` back to the caller protocol. Relay codecs are not used. + +The current streaming contract uses the normalized `LlmResponseChunk` +representation and does not preserve a raw provider-event envelope. +Common text, usage, finish-reason, and tool-call fields can be translated, but +unknown provider-specific fields in same-protocol SSE events are not guaranteed +to survive the decode/libsy/encode round trip. The streaming helpers also do not +expose the buffered translation engine's reject-lossy diagnostics, so unsupported +cross-protocol stream fields may be normalized or omitted. Do not claim +lossless streaming until Switchyard exposes both a raw provider-event +preservation contract and an explicit reject-lossy stream policy. + +## Configuration + +During release-candidate validation the manifest declares +`compat.native_api = "1"` and Relay `>=0.7.0-rc.4,<1.0`, and the Rust SDK uses +the exact published `0.7.0-rc.4` crate. Before release, move both lower bounds +to stable `0.7.0`. The manifest API value selects the released v1 plugin +contract; the binary is built against the V3 C host table shipped on the Relay +0.7 line, which is why the minimum Relay version is not 0.6. Rebuild the bundle +when changing SDK versions rather than assuming Rust dynamic-library +compatibility from the manifest value alone. + +A Relay project can configure a seeded weighted-random router as follows: + +```toml +version = 1 + +[[plugins.dynamic]] +manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 3 + +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 + +[plugins.dynamic.config.default_targets] +openai_chat = "fast" + +[plugins.dynamic.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 + +[plugins.dynamic.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" +``` + +Target map keys such as `fast` are stable semantic names visible to libsy. The +target binding is authoritative for the provider model, protocol, endpoint, +base URL, weight, and headers. Each `default_targets` key both enables that +inbound protocol and names its trusted fallback. + +`header_env` resolves target credentials in the plugin process at registration +time. Environment values must not appear in configuration, errors, routing +marks, spans, or debug output. The plugin does not inherit caller credentials +for managed calls. Each variable supplies the complete header value, so an +`authorization` value must include its scheme, such as `Bearer`. +Common credential headers, including `authorization` and `x-api-key`, are +rejected in static `headers` and must use `header_env`. Static headers remain +appropriate for non-secret routing or tenancy metadata. + +The Switchyard client does not follow provider redirects, applies a 10-second +connect timeout and a 120-second inactivity timeout, caps buffered provider +responses at 64 MiB, and caps retained HTTP error bodies at 64 KiB. Default +error display and Relay rejection messages do not include provider bodies. The +shared SSE decoder rejects an individual provider event above 8 MiB before its +line buffer can grow without bound. + +For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or +`openai_responses`; libsy's judge request uses a JSON-schema response format +that cannot be represented losslessly by Anthropic Messages. The remaining +classifier fields map directly to libsy's capability classifier configuration. + +Version-1 service configuration, decision-only execution, and observe-only +mode are rejected. + +## Build and bundle + +The crate is a source/build unit with `publish = false`. Operators install a +binary bundle rather than a Rust crate: + +```bash +cargo build --release -p switchyard-nemo-relay-plugin +python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library target/release/libswitchyard_nemo_relay_plugin.so \ + --output dist/switchyard-nemo-relay-plugin-linux-x86_64 +``` + +On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle +builder copies the shared library, materialized manifest, JSON schema, README, +LICENSE, and NOTICE, then writes `SHA256SUMS`. + +Install the materialized bundle with Relay's normal lifecycle commands: + +```bash +nemo-relay plugins validate /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins add /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins enable nvidia.switchyard +nemo-relay plugins inspect nvidia.switchyard +``` + +## Validation expectations + +Before release, validate both routers against buffered and streaming OpenAI +Chat, OpenAI Responses, and Anthropic Messages providers. The acceptance suite +must cover same- and supported cross-protocol routes, deterministic weighted +routing, independent runs, classifier weak and strong selections, retry +reselection, exhaustion, exactly-once fallback, stream commitment, empty +streams, late errors, cancellation, credential privacy, and unmanaged +pass-through. + +The tests must also prove that managed target traffic reaches the provider +through `switchyard-llm-client`, never through Relay's provider continuation, +and that no Switchyard service or health endpoint is involved. diff --git a/docs/index.md b/docs/index.md index 1dc5056cd..93814e9cc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages. | Run Claude Code, Codex, or OpenClaw through Switchyard | Launcher Path | [Install and launch an agent](getting_started.md#launcher-path) | | Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) | | Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) | +| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) | The Launcher Path installs the `switchyard` CLI and hosts the native Rust server through its packaged PyO3 binding. The Server Path builds and runs the @@ -30,3 +31,4 @@ standalone `switchyard-server` binary. - [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms - [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types - [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation +- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integration From fd151bd331678f043fa8025f59cb7982cf1e2a2e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 17:11:34 -0600 Subject: [PATCH 03/15] refactor(relay): emit cdylib only Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 1c94279dd..a95d27f48 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -13,7 +13,7 @@ rust-version.workspace = true publish = false [lib] -crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] [dependencies] async-trait.workspace = true From 0319bc33e9b42cdbc911e305463f0bf54f952bbb Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 17:33:53 -0600 Subject: [PATCH 04/15] refactor(relay): minimize plugin bundle Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 4 ++-- .../scripts/package_bundle.py | 16 +++------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index c0b984088..8f1416439 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -223,8 +223,8 @@ python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ ``` On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle -builder copies the shared library, materialized manifest, JSON schema, README, -LICENSE, and NOTICE, then writes `SHA256SUMS`. +builder creates the minimal Relay package: the shared library, a materialized +manifest with Relay's inline SHA-256 integrity digest, and the JSON schema. Install the materialized bundle with Relay's normal lifecycle commands: diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py index 168cbff4a..550f836ed 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Build an operator-facing NeMo Relay plugin bundle from a compiled cdylib.""" +"""Materialize the minimal Relay plugin bundle from a compiled cdylib.""" from __future__ import annotations @@ -11,7 +11,6 @@ from pathlib import Path CRATE_ROOT = Path(__file__).resolve().parents[1] -REPOSITORY_ROOT = CRATE_ROOT.parents[1] def digest(path: Path) -> str: @@ -24,7 +23,7 @@ def digest(path: Path) -> str: def main() -> None: - """Materialize a self-contained plugin bundle in an empty directory.""" + """Materialize a Relay-loadable plugin bundle in an empty directory.""" parser = argparse.ArgumentParser() parser.add_argument("--library", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) @@ -49,22 +48,13 @@ def main() -> None: artifact = output / library.name shutil.copy2(library, artifact) - for name in ("config.schema.json", "README.md"): - shutil.copy2(CRATE_ROOT / name, output / name) - for name in ("LICENSE", "NOTICE"): - shutil.copy2(REPOSITORY_ROOT / name, output / name) + shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") artifact_digest = digest(artifact) manifest = manifest.replace("", artifact.name) manifest = manifest.replace("", artifact_digest) (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") - checksums = [] - for path in sorted(output.iterdir(), key=lambda item: item.name): - if path.name != "SHA256SUMS" and path.is_file(): - checksums.append(f"{digest(path)} {path.name}") - (output / "SHA256SUMS").write_text("\n".join(checksums) + "\n", encoding="utf-8") - print(output) From f28309bdfb1379d4ccaf1bbcb0199d04e6b230b1 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 19:05:21 -0600 Subject: [PATCH 05/15] refactor(relay): use public Rust plugin SDK Signed-off-by: Bryan Bednarski --- Cargo.lock | 54 +++ Cargo.toml | 1 + .../switchyard-nemo-relay-plugin/Cargo.toml | 1 + crates/switchyard-nemo-relay-plugin/README.md | 125 +++-- .../src/executor.rs | 23 +- .../switchyard-nemo-relay-plugin/src/ffi.rs | 444 ------------------ .../switchyard-nemo-relay-plugin/src/lib.rs | 369 +++++---------- .../src/runtime.rs | 124 +++-- 8 files changed, 346 insertions(+), 795 deletions(-) delete mode 100644 crates/switchyard-nemo-relay-plugin/src/ffi.rs diff --git a/Cargo.lock b/Cargo.lock index 12e851537..fa5e11a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,6 +95,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -378,6 +390,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -403,6 +424,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "deadpool" version = "0.12.3" @@ -466,6 +493,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1256,6 +1303,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -2095,6 +2148,7 @@ dependencies = [ name = "switchyard-nemo-relay-plugin" version = "0.1.0" dependencies = [ + "async-channel", "async-trait", "futures-util", "http", diff --git a/Cargo.toml b/Cargo.toml index d21605dbd..0718ca3db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ repository = "https://github.com/NVIDIA-NeMo/Switchyard" rust-version = "1.96.1" [workspace.dependencies] +async-channel = "2" async-stream = "0.3" async-trait = "0.1" futures = "0.3" diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index a95d27f48..376778bdb 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -16,6 +16,7 @@ publish = false crate-type = ["cdylib"] [dependencies] +async-channel.workspace = true async-trait.workspace = true futures-util.workspace = true http = "1" diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 8f1416439..9de160c6e 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: Apache-2.0 This crate builds the external `nvidia.switchyard` native plugin. It embeds `switchyard-libsy`, drives `Algorithm::run_stream`, and uses `switchyard-llm-client` for provider HTTP calls. Managed calls use Relay's -generic asynchronous middleware hooks and do not require a targeted provider +public Rust LLM execution-intercept SDK and do not require a targeted provider continuation from Relay. The plugin uses NeMo Relay native API v1. It depends on the small @@ -59,47 +59,87 @@ This boundary has two important consequences: transport activity is therefore not represented as nested Relay LLM lifecycle events. Relay records the outer managed call and the plugin emits Switchyard routing marks; bridging Switchyard transport spans into Relay is - future work. With an exported Agent or turn scope, Relay 0.7 projects the - outer LLM span and routing-decision span as siblings in one OpenInference - trace. The native callback cannot parent an asynchronous decision mark - directly under the active LLM event; an embedded host that invokes an LLM - without an exported parent scope would therefore produce an orphan decision - span. + future work. Routing marks are delivered through the public SDK while Relay + is polling the active execution callback or stream, so they remain attached + to the current Relay scope stack. - Switchyard owns provider URLs, credentials, HTTP retry behavior, and translation for managed calls. Relay neither validates nor transports those target details. -## Native API v1 and asynchronous execution - -The manifest remains `compat.native_api = "1"`, but this rebuilt plugin requires -the generic C host-table v3 extension shipped by Relay 0.7. It registers through -v3's completion-based buffered and incremental streaming hooks, returns -`Pending` immediately, and performs libsy and HTTP work on a plugin-owned Tokio -runtime. Relay's runtime workers therefore do not wait synchronously for -provider I/O. - -The v3 stream hook retains Relay's bounded 32-event output queue. The plugin -retries a logical event when that queue is full, checks cancellation between -attempts, and releases every completion, continuation, stream, and captured -scope handle exactly once. Managed HTTP work is selected against Relay caller -cancellation, so dropping a buffered or streaming call drops its in-flight -request future. - -Unmanaged profiles use the same generic v3 continuation hooks for pass-through. -V3's downstream stream callback has continue/cancel control but no asynchronous -acknowledgement, so it cannot provide true end-to-end backpressure. The adapter -uses a nonblocking bridge capped at 8 MiB of queued, encoded event payloads; it -also caps the queue at 256 events and safely rejects a pass-through stream that -outruns either bound. The byte cap does not include transient JSON parsing or -in-memory representation overhead. Direct host forwarding without this bound would -still require a small generic Relay hook, but not a targeted-provider contract. -Managed Switchyard streams do not use this pass-through bridge. None of the -managed HTTP or routing operations requires a targeted LLM continuation. - -This is still a raw C boundary: Switchyard contains a small ownership adapter -for host strings and v3 handles because Relay 0.7 does not expose a safe Rust -facade for the generic async surface. That adapter is transport-independent; -all HTTP and routing behavior remains in Switchyard. +## Native API v1 and the public Rust SDK + +The manifest uses `compat.native_api = "1"`. The implementation registers with +`PluginContext::register_llm_execution_intercept` and +`PluginContext::register_llm_stream_execution_intercept`, receives typed +`LlmRequest`, `LlmNext`, and `LlmStreamNext` values, and returns the SDK's JSON +result or `LlmJsonStream`. The `nemo-relay-plugin` SDK owns the C callback +trampolines, host strings, continuation handles, panic containment, and native +stream lifecycle. Switchyard contains no raw C callback or host-table adapter. + +Switchyard performs libsy and provider I/O on a plugin-owned Tokio runtime. A +buffered SDK callback waits for that executor to finish. A managed streaming +callback returns a pull-based Rust iterator backed by a bounded 32-message +channel; the async producer waits when the consumer is slow. Dropping that +iterator closes the channel and aborts its in-flight routing task. + +The public native API v1 callback shapes do not provide full in-flight caller +cancellation. A buffered callback has no cancellation token, so a provider +request already in progress continues until it responds or reaches the client +timeout after the caller disconnects. Relay can cancel a streaming iterator +between pulls, but its synchronous `Iterator::next` call cannot be interrupted +while it is waiting for the next provider event. The configured 120-second +inactivity timeout bounds both cases. Supporting prompt disconnect propagation +would require an asynchronous public SDK callback or stream-polling contract; +the plugin does not bypass the SDK to recover that behavior. + +### Relay runtime capacity + +In Relay 0.7, the safe native LLM callbacks run directly in Relay's asynchronous +middleware future; Relay does not move them to Tokio's separate blocking pool. +Consequently, each active buffered Switchyard call occupies a normal Relay +Tokio worker while it waits for the plugin executor, and a streaming call +occupies a worker while its synchronous `Iterator::next` waits for the next +provider event. Exhausting those workers can delay unrelated Relay work. + +The Relay CLI constructs a Tokio multi-thread runtime without setting an +explicit worker count. Tokio therefore defaults to the number of CPU cores +available to the process and honors its `TOKIO_WORKER_THREADS` environment +variable. Operators can provide additional capacity while using this native API +v1 integration, for example: + +```bash +TOKIO_WORKER_THREADS=32 nemo-relay ... +``` + +This adjusts the normal asynchronous worker pool, not Tokio's blocking pool; +increasing `max_blocking_threads` does not address this integration. There is no +universal recommended value. Size the pool with headroom above the expected +number of concurrent managed calls and Relay's other work, then validate it +under representative provider latency and streaming concurrency. Increasing +the worker count mitigates starvation but does not restore cancellation or make +the synchronous boundary non-blocking. + +Embedded Relay hosts own their Tokio runtime and should configure the same +capacity explicitly: + +```rust +let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(32) + .enable_all() + .build()?; +``` + +The durable resolution is a public Relay SDK callback and stream-polling +contract that can yield while provider I/O is pending and receive caller +cancellation. Until that exists, keep the provider timeout bounded, provision +worker capacity, and load-test the intended concurrency rather than relying on +thread-pool growth alone. + +Unmanaged profiles call the typed `LlmNext` or `LlmStreamNext` continuation and +return its result directly. Managed streams use the bounded Switchyard channel; +unmanaged streams use Relay's SDK-owned pull iterator without an additional +bridge. None of the managed HTTP or routing operations requires a targeted LLM +continuation or direct access to Relay's C ABI. ## Supported routers @@ -144,11 +184,10 @@ preservation contract and an explicit reject-lossy stream policy. During release-candidate validation the manifest declares `compat.native_api = "1"` and Relay `>=0.7.0-rc.4,<1.0`, and the Rust SDK uses the exact published `0.7.0-rc.4` crate. Before release, move both lower bounds -to stable `0.7.0`. The manifest API value selects the released v1 plugin -contract; the binary is built against the V3 C host table shipped on the Relay -0.7 line, which is why the minimum Relay version is not 0.6. Rebuild the bundle -when changing SDK versions rather than assuming Rust dynamic-library -compatibility from the manifest value alone. +to stable `0.7.0`. The manifest API value selects Relay's released native +plugin contract; plugin authors use its safe Rust SDK rather than the underlying +C table directly. Rebuild the bundle when changing SDK versions rather than +assuming Rust dynamic-library compatibility from the manifest value alone. A Relay project can configure a seeded weighted-random router as follows: diff --git a/crates/switchyard-nemo-relay-plugin/src/executor.rs b/crates/switchyard-nemo-relay-plugin/src/executor.rs index 3dacb5e97..38b4d0f42 100644 --- a/crates/switchyard-nemo-relay-plugin/src/executor.rs +++ b/crates/switchyard-nemo-relay-plugin/src/executor.rs @@ -11,10 +11,10 @@ use tokio::task::AbortHandle; /// Plugin-owned async executor. /// -/// Relay's generic V3 host table lets the plugin return `Pending`, so neither -/// buffered nor streaming callbacks block Relay runtime workers. Keeping the -/// runtime on a dedicated thread also avoids entering Relay's Tokio runtime -/// from a separately linked cdylib. +/// The public native-plugin SDK uses synchronous Rust callbacks and pull-based +/// iterators at the dynamic-library boundary. Switchyard performs provider I/O +/// on this dedicated runtime rather than entering Relay's Tokio runtime from a +/// separately linked cdylib. #[derive(Clone)] pub(crate) struct PluginExecutor { inner: Arc, @@ -74,6 +74,20 @@ impl PluginExecutor { { self.inner.handle.spawn(future).abort_handle() } + + pub(crate) fn run(&self, future: F) -> Result + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let (sender, receiver) = mpsc::sync_channel(1); + self.inner.handle.spawn(async move { + let _ = sender.send(future.await); + }); + receiver + .recv() + .map_err(|_| "Switchyard HTTP runtime stopped before completing work".to_string()) + } } impl Drop for ExecutorInner { @@ -113,6 +127,7 @@ mod tests { #[test] fn executor_runs_buffered_and_spawned_work() { let executor = PluginExecutor::new().unwrap(); + assert_eq!(executor.run(async { 42 }).unwrap(), 42); let (sender, receiver) = mpsc::sync_channel(1); executor.spawn(async move { sender.send("done").unwrap(); diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs deleted file mode 100644 index f5ce094c1..000000000 --- a/crates/switchyard-nemo-relay-plugin/src/ffi.rs +++ /dev/null @@ -1,444 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Small ownership wrapper around Relay's generic C host-table v3 hooks. -//! -//! The plugin manifest remains native API v1. Relay 0.7 supplies the appended -//! v3 host table to rebuilt v1 plugins, which lets this crate return `Pending` -//! and settle work from its own runtime without a targeted-continuation ABI. - -use std::ffi::c_void; -use std::ptr; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Duration; - -use nemo_relay_plugin::{ - Json, LlmRequest, NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, - NemoRelayNativeAsyncNextStreamCb, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, - NemoRelayNativeHostApiV3, NemoRelayNativeScopeHandle, NemoRelayNativeString, NemoRelayStatus, -}; -use serde::Serialize; -use tokio::sync::{mpsc, oneshot}; - -const BACKPRESSURE_POLL: Duration = Duration::from_millis(1); -const CANCELLATION_POLL: Duration = Duration::from_millis(10); -const MAX_PASSTHROUGH_BUFFER_BYTES: usize = 8 * 1024 * 1024; -const MAX_PASSTHROUGH_BUFFER_EVENTS: usize = 256; - -pub(crate) struct HostString { - host: NemoRelayNativeHostApiV1, - ptr: *mut NemoRelayNativeString, -} - -// Host strings are immutable allocations owned by Relay's thread-safe host table. -unsafe impl Send for HostString {} - -impl HostString { - pub(crate) fn json( - host: &NemoRelayNativeHostApiV1, - value: &impl Serialize, - ) -> Result { - let value = serde_json::to_string(value).map_err(|error| error.to_string())?; - Self::text(host, &value) - } - - pub(crate) fn text(host: &NemoRelayNativeHostApiV1, value: &str) -> Result { - let mut ptr = ptr::null_mut(); - let status = unsafe { (host.string_new)(value.as_ptr(), value.len(), &mut ptr) }; - if status == NemoRelayStatus::Ok && !ptr.is_null() { - Ok(Self { host: *host, ptr }) - } else { - Err(format!("Relay host string allocation failed: {status:?}")) - } - } - - pub(crate) fn as_ptr(&self) -> *const NemoRelayNativeString { - self.ptr - } -} - -impl Drop for HostString { - fn drop(&mut self) { - unsafe { (self.host.string_free)(self.ptr) }; - } -} - -pub(crate) fn read_string( - host: &NemoRelayNativeHostApiV1, - value: *const NemoRelayNativeString, -) -> Result { - if value.is_null() { - return Err("Relay passed a null native string".into()); - } - let len = unsafe { (host.string_len)(value) }; - let data = unsafe { (host.string_data)(value) }; - if data.is_null() && len != 0 { - return Err("Relay passed an invalid native string".into()); - } - let bytes = if len == 0 { - &[][..] - } else { - unsafe { std::slice::from_raw_parts(data, len) } - }; - std::str::from_utf8(bytes) - .map(str::to_owned) - .map_err(|error| error.to_string()) -} - -pub(crate) fn read_json( - host: &NemoRelayNativeHostApiV1, - value: *const NemoRelayNativeString, -) -> Result { - serde_json::from_str(&read_string(host, value)?).map_err(|error| error.to_string()) -} - -/// Captures the current Relay scope as an explicit event parent. -/// -/// Async plugin work runs on a plugin-owned thread, so relying on thread-local -/// scope state would orphan its marks. The host handle is a cloned scope handle -/// and remains valid until this guard is dropped. -pub(crate) struct ParentScope { - host: NemoRelayNativeHostApiV1, - ptr: *mut NemoRelayNativeScopeHandle, -} - -unsafe impl Send for ParentScope {} -unsafe impl Sync for ParentScope {} - -impl ParentScope { - pub(crate) fn capture(host: &NemoRelayNativeHostApiV1) -> Option { - let mut ptr = ptr::null_mut(); - let status = unsafe { (host.scope_get_current)(&mut ptr) }; - (status == NemoRelayStatus::Ok && !ptr.is_null()).then_some(Self { host: *host, ptr }) - } - - pub(crate) fn emit_mark(&self, name: &str, data: &Json, metadata: &Json) -> Result<(), String> { - let name = HostString::text(&self.host, name)?; - let data = HostString::json(&self.host, data)?; - let metadata = HostString::json(&self.host, metadata)?; - let status = unsafe { - (self.host.emit_mark)( - name.as_ptr(), - self.ptr, - data.as_ptr(), - metadata.as_ptr(), - ptr::null(), - ) - }; - if status == NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!( - "Relay rejected Switchyard routing mark: {status:?}" - )) - } - } -} - -impl Drop for ParentScope { - fn drop(&mut self) { - unsafe { (self.host.scope_handle_free)(self.ptr) }; - } -} - -pub(crate) fn invoke_next_buffered( - host: &NemoRelayNativeHostApiV3, - next: usize, - completion: usize, - request: &LlmRequest, -) -> Result<(), String> { - let request = HostString::json(&host.v1, request)?; - let status = unsafe { - (host.async_next_invoke)( - next as *const NemoRelayNativeAsyncNext, - request.as_ptr(), - completion as *const NemoRelayNativeAsyncCompletion, - ) - }; - if status == NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!("Relay rejected buffered pass-through: {status:?}")) - } -} - -enum DownstreamStreamItem { - Chunk { value: Json, encoded_bytes: usize }, -} - -struct DownstreamStreamState { - host: NemoRelayNativeHostApiV1, - sender: mpsc::Sender, - terminal: Option>>, - queued_bytes: Arc, -} - -pub(crate) async fn invoke_next_stream( - host: &NemoRelayNativeHostApiV3, - next: usize, - output: usize, - request: &LlmRequest, -) -> Result<(), String> { - let request = HostString::json(&host.v1, request)?; - let (sender, mut receiver) = mpsc::channel(MAX_PASSTHROUGH_BUFFER_EVENTS); - let (terminal, terminal_result) = oneshot::channel(); - let queued_bytes = Arc::new(AtomicUsize::new(0)); - let state = Box::into_raw(Box::new(DownstreamStreamState { - host: host.v1, - sender, - terminal: Some(terminal), - queued_bytes: Arc::clone(&queued_bytes), - })) - .cast::(); - let status = unsafe { - (host.async_next_invoke_stream)( - next as *const NemoRelayNativeAsyncNext, - request.as_ptr(), - output as *const NemoRelayNativeAsyncStream, - downstream_stream_result as NemoRelayNativeAsyncNextStreamCb, - state, - ) - }; - if status != NemoRelayStatus::Ok { - unsafe { drop(Box::from_raw(state.cast::())) }; - return Err(format!("Relay rejected streaming pass-through: {status:?}")); - } - - while let Some(item) = receiver.recv().await { - match item { - DownstreamStreamItem::Chunk { - value, - encoded_bytes, - } => { - let result = push_stream(host, output, &value).await; - queued_bytes.fetch_sub(encoded_bytes, Ordering::AcqRel); - result?; - } - } - } - terminal_result - .await - .unwrap_or_else(|_| Err("Relay dropped the streaming pass-through callback".into())) -} - -unsafe extern "C" fn downstream_stream_result( - user_data: *mut c_void, - chunk_json: *const NemoRelayNativeString, - error: *const NemoRelayNativeString, - done: bool, -) -> bool { - if !error.is_null() { - let state = unsafe { Box::from_raw(user_data.cast::()) }; - let error = read_string(&state.host, error) - .unwrap_or_else(|_| "Relay streaming pass-through failed".into()); - settle_downstream_stream(state, Err(error)); - return false; - } - if done { - let state = unsafe { Box::from_raw(user_data.cast::()) }; - settle_downstream_stream(state, Ok(())); - return false; - } - - let state = unsafe { &*user_data.cast::() }; - let parsed = read_string(&state.host, chunk_json).and_then(|encoded| { - let encoded_bytes = encoded.len(); - let value = serde_json::from_str(&encoded).map_err(|error| error.to_string())?; - Ok((value, encoded_bytes)) - }); - let (value, encoded_bytes) = match parsed { - Ok(parsed) => parsed, - Err(error) => { - let state = unsafe { Box::from_raw(user_data.cast::()) }; - settle_downstream_stream(state, Err(error)); - return false; - } - }; - if !reserve_buffer_bytes(&state.queued_bytes, encoded_bytes) { - let state = unsafe { Box::from_raw(user_data.cast::()) }; - settle_downstream_stream( - state, - Err(format!( - "Relay streaming pass-through exceeded its {}-byte queued payload limit", - MAX_PASSTHROUGH_BUFFER_BYTES - )), - ); - return false; - } - - match state.sender.try_send(DownstreamStreamItem::Chunk { - value, - encoded_bytes, - }) { - Ok(()) => true, - Err(error) => { - let (item, message) = match error { - mpsc::error::TrySendError::Full(item) => ( - item, - format!( - "Relay streaming pass-through exceeded its {MAX_PASSTHROUGH_BUFFER_EVENTS}-event queue" - ), - ), - mpsc::error::TrySendError::Closed(item) => ( - item, - "Relay dropped the streaming pass-through receiver".into(), - ), - }; - let encoded_bytes = item.encoded_bytes(); - state - .queued_bytes - .fetch_sub(encoded_bytes, Ordering::AcqRel); - let state = unsafe { Box::from_raw(user_data.cast::()) }; - settle_downstream_stream(state, Err(message)); - false - } - } -} - -impl DownstreamStreamItem { - fn encoded_bytes(&self) -> usize { - match self { - Self::Chunk { encoded_bytes, .. } => *encoded_bytes, - } - } -} - -fn settle_downstream_stream(mut state: Box, result: Result<(), String>) { - if let Some(terminal) = state.terminal.take() { - let _ = terminal.send(result); - } -} - -fn reserve_buffer_bytes(queued: &AtomicUsize, encoded_bytes: usize) -> bool { - queued - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { - current - .checked_add(encoded_bytes) - .filter(|next| *next <= MAX_PASSTHROUGH_BUFFER_BYTES) - }) - .is_ok() -} - -pub(crate) async fn wait_for_completion_cancellation( - host: &NemoRelayNativeHostApiV3, - completion: usize, -) { - while !completion_cancelled(host, completion as *const NemoRelayNativeAsyncCompletion) { - tokio::time::sleep(CANCELLATION_POLL).await; - } -} - -pub(crate) async fn wait_for_stream_cancellation(host: &NemoRelayNativeHostApiV3, stream: usize) { - while !unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } - { - tokio::time::sleep(CANCELLATION_POLL).await; - } -} - -pub(crate) fn completion_cancelled( - host: &NemoRelayNativeHostApiV3, - completion: *const NemoRelayNativeAsyncCompletion, -) -> bool { - unsafe { (host.async_completion_is_cancelled)(completion) } -} - -pub(crate) fn resolve_completion( - host: &NemoRelayNativeHostApiV3, - completion: *const NemoRelayNativeAsyncCompletion, - value: &Json, -) -> NemoRelayStatus { - match HostString::json(&host.v1, value) { - Ok(value) => unsafe { (host.async_completion_resolve_json)(completion, value.as_ptr()) }, - Err(_) => NemoRelayStatus::Internal, - } -} - -pub(crate) fn reject_completion( - host: &NemoRelayNativeHostApiV3, - completion: *const NemoRelayNativeAsyncCompletion, - message: &str, -) -> NemoRelayStatus { - match HostString::text(&host.v1, message) { - Ok(message) => unsafe { (host.async_completion_reject)(completion, message.as_ptr()) }, - Err(_) => NemoRelayStatus::Internal, - } -} - -pub(crate) async fn push_stream( - host: &NemoRelayNativeHostApiV3, - stream: usize, - value: &Json, -) -> Result<(), String> { - let value = HostString::json(&host.v1, value)?; - loop { - if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } - { - return Err("Relay caller cancelled the output stream".into()); - } - match unsafe { - (host.async_stream_push_json)( - stream as *const NemoRelayNativeAsyncStream, - value.as_ptr(), - ) - } { - NemoRelayStatus::Ok => return Ok(()), - // Native API v1 reports its bounded queue's WouldBlock state as Internal. - NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, - status => return Err(format!("Relay rejected output stream event: {status:?}")), - } - } -} - -pub(crate) fn finish_stream( - host: &NemoRelayNativeHostApiV3, - stream: *const NemoRelayNativeAsyncStream, -) -> NemoRelayStatus { - unsafe { (host.async_stream_finish)(stream) } -} - -pub(crate) async fn reject_stream( - host: &NemoRelayNativeHostApiV3, - stream: usize, - message: &str, -) -> NemoRelayStatus { - let Ok(message) = HostString::text(&host.v1, message) else { - return NemoRelayStatus::Internal; - }; - loop { - if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } - { - return NemoRelayStatus::InvalidArg; - } - match unsafe { - (host.async_stream_reject)( - stream as *const NemoRelayNativeAsyncStream, - message.as_ptr(), - ) - } { - NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, - status => return status, - } - } -} - -pub(crate) unsafe fn release_completion( - host: &NemoRelayNativeHostApiV3, - completion: *const NemoRelayNativeAsyncCompletion, -) { - unsafe { (host.async_completion_release)(completion) }; -} - -pub(crate) unsafe fn release_next( - host: &NemoRelayNativeHostApiV3, - next: *const NemoRelayNativeAsyncNext, -) { - unsafe { (host.async_next_release)(next) }; -} - -pub(crate) unsafe fn release_stream( - host: &NemoRelayNativeHostApiV3, - stream: *const NemoRelayNativeAsyncStream, -) { - unsafe { (host.async_stream_release)(stream) }; -} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs index 8f6ba68b6..85fd634a3 100644 --- a/crates/switchyard-nemo-relay-plugin/src/lib.rs +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -4,39 +4,28 @@ mod client; mod config; mod executor; -mod ffi; mod runtime; mod translation; -use std::ffi::c_void; -use std::mem; use std::panic::AssertUnwindSafe; use std::sync::Arc; use futures_util::FutureExt; use nemo_relay_plugin::{ - ConfigDiagnostic, DiagnosticLevel, Json, NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE, - NativePlugin, NemoRelayNativeAsyncCallbackState, NemoRelayNativeAsyncCompletion, - NemoRelayNativeAsyncMiddlewareKind, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, - NemoRelayNativeHostApiV3, NemoRelayNativeString, NemoRelayStatus, PluginContext, + ConfigDiagnostic, DiagnosticLevel, Json, LlmJsonStream, LlmNext, LlmRequest, LlmStreamNext, + NativePlugin, PluginContext, PluginRuntime, }; -use serde::Deserialize; use serde_json::Map; +use tokio::task::AbortHandle; use crate::config::SwitchyardConfig; use crate::executor::PluginExecutor; -use crate::runtime::SwitchyardRuntime; - -#[derive(Deserialize)] -struct Invocation { - name: String, - request: nemo_relay_plugin::LlmRequest, -} +use crate::runtime::{RoutingMark, StreamMessage, SwitchyardRuntime}; struct CallbackState { - host: NemoRelayNativeHostApiV3, runtime: Arc, executor: PluginExecutor, + relay: PluginRuntime, } #[derive(Default)] @@ -69,77 +58,26 @@ impl NativePlugin for SwitchyardPlugin { plugin_config: &Map, ctx: &mut PluginContext<'_>, ) -> nemo_relay_plugin::Result<()> { - let host_v1 = ctx.host_api(); - if host_v1.abi_version < NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE - || host_v1.struct_size < mem::size_of::() - { - return Err( - "Switchyard requires Relay 0.7 or newer with the generic asynchronous native host table" - .into(), - ); - } - let host = unsafe { *(host_v1 as *const _ as *const NemoRelayNativeHostApiV3) }; let config = parse_config(plugin_config)?; let priority = config.priority; let state = Arc::new(CallbackState { - host, runtime: Arc::new(SwitchyardRuntime::new(config)?), executor: PluginExecutor::new()?, + relay: ctx.runtime(), }); - register_buffered(ctx, priority, Arc::clone(&state))?; - register_stream(ctx, priority, state)?; - Ok(()) - } -} - -fn register_buffered( - ctx: &mut PluginContext<'_>, - priority: i32, - state: Arc, -) -> Result<(), String> { - let user_data = Box::into_raw(Box::new(state)).cast::(); - let status = unsafe { - ctx.register_async_middleware_raw( - NemoRelayNativeAsyncMiddlewareKind::LlmExecutionIntercept, + let buffered = Arc::clone(&state); + ctx.register_llm_execution_intercept( "switchyard.run_stream.buffered", priority, - false, - buffered_callback, - user_data, - Some(free_callback_state), - ) - }; - if status == NemoRelayStatus::Ok { - Ok(()) - } else { - Err(format!( - "failed to register Switchyard buffered execution: {status:?}" - )) - } -} - -fn register_stream( - ctx: &mut PluginContext<'_>, - priority: i32, - state: Arc, -) -> Result<(), String> { - let user_data = Box::into_raw(Box::new(state)).cast::(); - let status = unsafe { - ctx.register_async_stream_middleware_raw( + move |name, request, next| execute_buffered(&buffered, name, request, next), + )?; + ctx.register_llm_stream_execution_intercept( "switchyard.run_stream.streaming", priority, - stream_callback, - user_data, - Some(free_callback_state), - ) - }; - if status == NemoRelayStatus::Ok { + move |name, request, next| execute_stream(&state, name, request, next), + )?; Ok(()) - } else { - Err(format!( - "failed to register Switchyard streaming execution: {status:?}" - )) } } @@ -159,205 +97,112 @@ fn parse_config(plugin_config: &Map) -> Result>())) }; - } +fn execute_buffered( + state: &CallbackState, + name: &str, + request: LlmRequest, + next: LlmNext<'_>, +) -> nemo_relay_plugin::Result { + let Some(inbound) = state.runtime.managed_protocol(name) else { + return next.call(request); + }; + let request = state.runtime.decode_request(inbound, &request, false)?; + let runtime = Arc::clone(&state.runtime); + let (result, marks) = state.executor.run(async move { + let mut marks = Vec::new(); + let result = AssertUnwindSafe(runtime.execute_buffered(inbound, request, &mut marks)) + .catch_unwind() + .await + .unwrap_or_else(|_| Err("Switchyard buffered execution panicked".into())); + (result, marks) + })?; + emit_marks(&state.relay, marks); + result } -unsafe extern "C" fn buffered_callback( - user_data: *mut c_void, - invocation_json: *const NemoRelayNativeString, - next: *const NemoRelayNativeAsyncNext, - completion: *const NemoRelayNativeAsyncCompletion, -) -> u32 { - if user_data.is_null() || completion.is_null() || next.is_null() { - return NemoRelayNativeAsyncCallbackState::Complete as u32; - } - let state = unsafe { &*user_data.cast::>() }.clone(); - let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { - serde_json::from_value::(value).map_err(|error| error.to_string()) - }); - let next = next as usize; - let completion = completion as usize; - let invocation = match invocation { - Ok(invocation) => invocation, - Err(error) => { - let _ = ffi::reject_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - &format!("invalid Relay LLM invocation: {error}"), - ); - unsafe { - ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); - ffi::release_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - ); - } - return NemoRelayNativeAsyncCallbackState::Pending as u32; - } +fn execute_stream( + state: &CallbackState, + name: &str, + request: LlmRequest, + next: LlmStreamNext<'_>, +) -> nemo_relay_plugin::Result { + let Some(inbound) = state.runtime.managed_protocol(name) else { + return Ok(Box::new(next.call(request)?)); }; - let Some(inbound) = state.runtime.managed_protocol(&invocation.name) else { - if let Err(error) = - ffi::invoke_next_buffered(&state.host, next, completion, &invocation.request) - { - let _ = ffi::reject_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - &error, - ); - } - unsafe { - ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); - ffi::release_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - ); - } - return NemoRelayNativeAsyncCallbackState::Pending as u32; - }; - let request = match state - .runtime - .decode_request(inbound, &invocation.request, false) - { - Ok(request) => request, - Err(error) => { - let _ = ffi::reject_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - &error, - ); - unsafe { - ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); - ffi::release_completion( - &state.host, - completion as *const NemoRelayNativeAsyncCompletion, - ); - } - return NemoRelayNativeAsyncCallbackState::Pending as u32; - } - }; - let parent = ffi::ParentScope::capture(&state.host.v1); - let task_state = Arc::clone(&state); - state.executor.spawn(async move { - let execution = AssertUnwindSafe(task_state.runtime.execute_buffered( - inbound, - request, - parent.as_ref(), - )) - .catch_unwind(); - tokio::pin!(execution); - let result = tokio::select! { - biased; - () = ffi::wait_for_completion_cancellation(&task_state.host, completion) => None, - result = &mut execution => Some( - result.unwrap_or_else(|_| Err("Switchyard buffered execution panicked".into())) - ), - }; - - let completion_ptr = completion as *const NemoRelayNativeAsyncCompletion; - if let Some(result) = result { - match result { - Ok(response) => { - let _ = ffi::resolve_completion(&task_state.host, completion_ptr, &response); - } - Err(error) => { - let _ = ffi::reject_completion(&task_state.host, completion_ptr, &error); - } - } - } - unsafe { - ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); - ffi::release_completion(&task_state.host, completion_ptr); + let request = state.runtime.decode_request(inbound, &request, true)?; + let (sender, receiver) = async_channel::bounded(32); + let runtime = Arc::clone(&state.runtime); + let task = state.executor.spawn(async move { + let result = AssertUnwindSafe(runtime.execute_stream(inbound, request, &sender)) + .catch_unwind() + .await + .unwrap_or_else(|_| Err("Switchyard streaming execution panicked".into())); + if let Err(error) = result { + let _ = sender.send(StreamMessage::Error(error)).await; } }); - NemoRelayNativeAsyncCallbackState::Pending as u32 + Ok(Box::new(SwitchyardStream { + receiver, + task: Some(task), + relay: state.relay.clone(), + finished: false, + })) } -unsafe extern "C" fn stream_callback( - user_data: *mut c_void, - invocation_json: *const NemoRelayNativeString, - next: *const NemoRelayNativeAsyncNext, - output: *const NemoRelayNativeAsyncStream, -) -> u32 { - if user_data.is_null() || output.is_null() || next.is_null() { - return NemoRelayNativeAsyncCallbackState::Complete as u32; +fn emit_marks(relay: &PluginRuntime, marks: Vec) { + for mark in marks { + emit_mark(relay, mark); } - let state = unsafe { &*user_data.cast::>() }.clone(); - let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { - serde_json::from_value::(value).map_err(|error| error.to_string()) - }); - let managed_protocol = invocation - .as_ref() - .ok() - .and_then(|invocation| state.runtime.managed_protocol(&invocation.name)); - let parent = managed_protocol.and_then(|_| ffi::ParentScope::capture(&state.host.v1)); - let next = next as usize; - let output = output as usize; - let task_state = Arc::clone(&state); - state.executor.spawn(async move { - let execution = AssertUnwindSafe(async { - match invocation { - Ok(invocation) => { - if let Some(inbound) = managed_protocol { - match task_state - .runtime - .decode_request(inbound, &invocation.request, true) - { - Ok(request) => { - task_state - .runtime - .execute_stream( - &task_state.host, - output, - inbound, - request, - parent.as_ref(), - ) - .await - } - Err(error) => Err(error), - } - } else { - ffi::invoke_next_stream(&task_state.host, next, output, &invocation.request) - .await - } - } - Err(error) => Err(format!("invalid Relay LLM stream invocation: {error}")), - } - }) - .catch_unwind(); - tokio::pin!(execution); - let result = tokio::select! { - biased; - () = ffi::wait_for_stream_cancellation(&task_state.host, output) => None, - result = &mut execution => Some( - result.unwrap_or_else(|_| Err("Switchyard streaming execution panicked".into())) - ), - }; +} - match result { - Some(Ok(())) => { - let _ = ffi::finish_stream( - &task_state.host, - output as *const NemoRelayNativeAsyncStream, - ); - } - Some(Err(error)) => { - let _ = ffi::reject_stream(&task_state.host, output, &error).await; +fn emit_mark(relay: &PluginRuntime, mark: RoutingMark) { + if let Err(error) = relay.emit_mark(&mark.name, Some(&mark.data), Some(&mark.metadata)) { + eprintln!( + "Switchyard could not emit routing mark {:?}: {error}", + mark.name + ); + } +} + +struct SwitchyardStream { + receiver: async_channel::Receiver, + task: Option, + relay: PluginRuntime, + finished: bool, +} + +impl Iterator for SwitchyardStream { + type Item = nemo_relay_plugin::Result; + + fn next(&mut self) -> Option { + loop { + match self.receiver.recv_blocking() { + Ok(StreamMessage::Mark(mark)) => emit_mark(&self.relay, mark), + Ok(StreamMessage::Event(event)) => return Some(Ok(event)), + Ok(StreamMessage::Error(error)) => { + self.finished = true; + self.task.take(); + return Some(Err(error)); + } + Err(_) => { + self.finished = true; + self.task.take(); + return None; + } } - None => {} } - unsafe { - ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); - ffi::release_stream( - &task_state.host, - output as *const NemoRelayNativeAsyncStream, - ); + } +} + +impl Drop for SwitchyardStream { + fn drop(&mut self) { + self.receiver.close(); + if !self.finished + && let Some(task) = self.task.take() + { + task.abort(); } - }); - NemoRelayNativeAsyncCallbackState::Pending as u32 + } } nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index f9cfea69f..b9d719a6d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use futures_util::{StreamExt, stream}; -use nemo_relay_plugin::{Json, LlmRequest as RelayRequest, NemoRelayNativeHostApiV3}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; use serde_json::{Map, json}; use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Step}; use switchyard_protocol::{ @@ -15,8 +15,21 @@ use switchyard_protocol::{ use switchyard_translation::{TranslationEngine, encode_stream}; use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; -use crate::ffi::ParentScope; -use crate::{ffi, translation}; +use crate::translation; + +#[derive(Debug)] +pub(crate) struct RoutingMark { + pub(crate) name: String, + pub(crate) data: Json, + pub(crate) metadata: Json, +} + +#[derive(Debug)] +pub(crate) enum StreamMessage { + Mark(RoutingMark), + Event(Json), + Error(String), +} pub(crate) struct SwitchyardRuntime { max_retries: u32, @@ -67,22 +80,19 @@ impl SwitchyardRuntime { &self, inbound: WireFormat, request: Request, - parent: Option<&ParentScope>, + marks: &mut Vec, ) -> Result { let metadata = identity_metadata(request.metadata.as_ref()); let max_attempts = self.max_retries + 1; let mut attempt = 1; loop { self.mark( - parent, + marks, "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); - match self - .drive(request.clone(), attempt, parent, &metadata) - .await - { + match self.drive(request.clone(), attempt, marks, &metadata).await { Ok(response) => { let LlmResponse::Agg(response) = response.llm_response else { return Err("libsy returned a stream for a buffered request".into()); @@ -91,7 +101,7 @@ impl SwitchyardRuntime { } Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { self.mark( - parent, + marks, "switchyard.routing.retry", failure_mark_data(attempt, &failure), &metadata, @@ -100,13 +110,13 @@ impl SwitchyardRuntime { } Err(failure) => { self.mark( - parent, + marks, "switchyard.routing.error", failure_mark_data(attempt, &failure), &metadata, ); return self - .fallback_buffered(inbound, request, parent, &metadata) + .fallback_buffered(inbound, request, marks, &metadata) .await; } } @@ -115,51 +125,52 @@ impl SwitchyardRuntime { pub(crate) async fn execute_stream( &self, - host: &NemoRelayNativeHostApiV3, - output: usize, inbound: WireFormat, request: Request, - parent: Option<&ParentScope>, + output: &async_channel::Sender, ) -> Result<(), String> { let metadata = identity_metadata(request.metadata.as_ref()); let max_attempts = self.max_retries + 1; let mut attempt = 1; + let mut marks = Vec::new(); loop { self.mark( - parent, + &mut marks, "switchyard.routing.requested", json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); let (response, fallback_used) = match self - .drive(request.clone(), attempt, parent, &metadata) + .drive(request.clone(), attempt, &mut marks, &metadata) .await { Ok(response) => (response, false), Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { self.mark( - parent, + &mut marks, "switchyard.routing.retry", failure_mark_data(attempt, &failure), &metadata, ); attempt += 1; + send_marks(output, &mut marks).await?; continue; } Err(failure) => { self.mark( - parent, + &mut marks, "switchyard.routing.error", failure_mark_data(attempt, &failure), &metadata, ); ( - self.fallback_response(inbound, request.clone(), parent, &metadata) + self.fallback_response(inbound, request.clone(), &mut marks, &metadata) .await?, true, ) } }; + send_marks(output, &mut marks).await?; let mut events = match returned_events(response, inbound).await { Ok(events) => events, @@ -169,24 +180,26 @@ impl SwitchyardRuntime { && attempt < max_attempts => { self.mark( - parent, + &mut marks, "switchyard.routing.retry", failure_mark_data(attempt, &failure), &metadata, ); attempt += 1; + send_marks(output, &mut marks).await?; continue; } Err(failure) if !fallback_used => { self.mark( - parent, + &mut marks, "switchyard.routing.error", failure_mark_data(attempt, &failure), &metadata, ); let fallback = self - .fallback_response(inbound, request.clone(), parent, &metadata) + .fallback_response(inbound, request.clone(), &mut marks, &metadata) .await?; + send_marks(output, &mut marks).await?; returned_events(fallback, inbound) .await .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? @@ -200,7 +213,7 @@ impl SwitchyardRuntime { while let Some(item) = events.next().await { match item { Ok(event) => { - ffi::push_stream(host, output, &event).await?; + send_event(output, event).await?; committed = true; } Err(failure) @@ -210,24 +223,26 @@ impl SwitchyardRuntime { && attempt < max_attempts => { self.mark( - parent, + &mut marks, "switchyard.routing.retry", failure_mark_data(attempt, &failure), &metadata, ); attempt += 1; + send_marks(output, &mut marks).await?; break; } Err(failure) if !fallback_used && !committed => { self.mark( - parent, + &mut marks, "switchyard.routing.error", failure_mark_data(attempt, &failure), &metadata, ); let fallback = self - .fallback_response(inbound, request.clone(), parent, &metadata) + .fallback_response(inbound, request.clone(), &mut marks, &metadata) .await?; + send_marks(output, &mut marks).await?; let mut fallback = returned_events(fallback, inbound).await.map_err(|error| { public_libsy_failure("trusted fallback stream", &error) @@ -236,7 +251,7 @@ impl SwitchyardRuntime { let event = item.map_err(|error| { public_libsy_failure("trusted fallback stream", &error) })?; - ffi::push_stream(host, output, &event).await?; + send_event(output, event).await?; } return Ok(()); } @@ -245,11 +260,12 @@ impl SwitchyardRuntime { } Err(failure) => { self.mark( - parent, + &mut marks, "switchyard.routing.error", failure_mark_data(attempt, &failure), &metadata, ); + send_marks(output, &mut marks).await?; return Err(public_libsy_failure( "Switchyard stream failed after response commitment", &failure, @@ -270,7 +286,7 @@ impl SwitchyardRuntime { &self, request: Request, attempt: u32, - parent: Option<&ParentScope>, + marks: &mut Vec, mark_metadata: &Json, ) -> Result { let context = context_from_metadata(request.metadata.as_ref()); @@ -278,7 +294,7 @@ impl SwitchyardRuntime { while let Some(step) = steps.next().await { match step { Ok(Step::Decision(decision)) => { - self.emit_decision(parent, decision.as_ref(), attempt, mark_metadata); + self.emit_decision(marks, decision.as_ref(), attempt, mark_metadata); } Ok(Step::CallLlm(call)) => self.serve_call(*call).await?, Ok(Step::ReturnToAgent(response)) => return Ok(*response), @@ -310,11 +326,11 @@ impl SwitchyardRuntime { &self, inbound: WireFormat, request: Request, - parent: Option<&ParentScope>, + marks: &mut Vec, metadata: &Json, ) -> Result { let response = self - .fallback_response(inbound, request, parent, metadata) + .fallback_response(inbound, request, marks, metadata) .await?; let LlmResponse::Agg(response) = response.llm_response else { return Err("trusted fallback returned a stream for a buffered request".into()); @@ -326,13 +342,13 @@ impl SwitchyardRuntime { &self, inbound: WireFormat, request: Request, - parent: Option<&ParentScope>, + marks: &mut Vec, metadata: &Json, ) -> Result { let target_name = self.default_target(inbound)?; let target = self.target(target_name)?; self.mark( - parent, + marks, "switchyard.routing.fallback", json!({"selected_target": target_name}), metadata, @@ -362,22 +378,23 @@ impl SwitchyardRuntime { .ok_or_else(|| format!("managed protocol {protocol} has no default target")) } - fn mark(&self, parent: Option<&ParentScope>, name: &str, data: Json, metadata: &Json) { - let Some(parent) = parent else { return }; - if let Err(error) = parent.emit_mark(name, &data, metadata) { - eprintln!("Switchyard could not emit routing mark {name:?}: {error}"); - } + fn mark(&self, marks: &mut Vec, name: &str, data: Json, metadata: &Json) { + marks.push(RoutingMark { + name: name.to_string(), + data, + metadata: metadata.clone(), + }); } fn emit_decision( &self, - parent: Option<&ParentScope>, + marks: &mut Vec, decision: &dyn Decision, attempt: u32, metadata: &Json, ) { self.mark( - parent, + marks, "switchyard.routing.decision", json!({ "algorithm": self.algorithm.name(), @@ -392,6 +409,29 @@ impl SwitchyardRuntime { } } +async fn send_marks( + output: &async_channel::Sender, + marks: &mut Vec, +) -> Result<(), String> { + for mark in marks.drain(..) { + output + .send(StreamMessage::Mark(mark)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string())?; + } + Ok(()) +} + +async fn send_event( + output: &async_channel::Sender, + event: Json, +) -> Result<(), String> { + output + .send(StreamMessage::Event(event)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string()) +} + type ReturnedEventStream = std::pin::Pin> + Send>>; From 1815a4554c6cec1478e8f399285ed0607da18936 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 20:05:21 -0600 Subject: [PATCH 06/15] refactor(relay): trim plugin-only surface Signed-off-by: Bryan Bednarski --- .../src/client.rs | 11 -------- .../src/config.rs | 13 +-------- .../src/runtime.rs | 28 ++++++------------- 3 files changed, 10 insertions(+), 42 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs index c8e00a32b..ae15ab8fe 100644 --- a/crates/switchyard-nemo-relay-plugin/src/client.rs +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -73,11 +73,6 @@ impl TargetClient { metadata.http_headers = None; request } - - #[cfg(test)] - fn provider_model(&self) -> &str { - &self.provider_model - } } #[async_trait] @@ -161,12 +156,6 @@ mod tests { ); } - #[test] - fn semantic_selection_does_not_replace_the_provider_model() { - let client = client(WireFormat::OpenAiChat); - assert_eq!(client.provider_model(), "provider/model"); - } - #[test] fn only_anthropic_targets_advertise_count_tokens() { assert!(client(WireFormat::AnthropicMessages).supports_count_tokens()); diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 8b28c6abe..1e1538aa8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -162,8 +162,6 @@ enum AlgorithmConfig { classifier_target: String, weak_target: String, strong_target: String, - #[serde(default)] - escalation: Option, #[serde(flatten)] config: TaskClassifierConfig, }, @@ -296,15 +294,8 @@ impl SwitchyardConfig { classifier_target, weak_target, strong_target, - escalation, config, } => { - if escalation.is_some() { - return Err( - "llm_classifier escalation mode is not supported by this plugin version" - .into(), - ); - } let classifier_binding = self.targets.get(classifier_target).ok_or_else(|| { format!("algorithm target {classifier_target:?} is not configured") })?; @@ -357,7 +348,7 @@ fn validate_dispatch_url( } // The current switchyard-llm-client accepts provider base URLs and complete - // canonical endpoints. Reject a custom terminal route here instead of + // canonical endpoints. Reject a custom terminal route to avoid // allowing Backend::url() to append another provider suffix silently. let expected_suffix = match protocol { WireFormat::OpenAiChat => "/chat/completions", @@ -686,7 +677,6 @@ mod tests { classifier_target: "anthropic".into(), weak_target: "responses".into(), strong_target: "chat".into(), - escalation: None, config: TaskClassifierConfig { base_threshold: 0.5, ..Default::default() @@ -743,7 +733,6 @@ mod tests { classifier_target: "chat".into(), weak_target: "responses".into(), strong_target: "anthropic".into(), - escalation: None, config: TaskClassifierConfig { base_threshold: 1.1, ..Default::default() diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index b9d719a6d..bb86751e8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -115,9 +115,15 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); - return self - .fallback_buffered(inbound, request, marks, &metadata) - .await; + let response = self + .fallback_response(inbound, request, marks, &metadata) + .await?; + let LlmResponse::Agg(response) = response.llm_response else { + return Err( + "trusted fallback returned a stream for a buffered request".into() + ); + }; + return translation::encode_response(&self.translation, inbound, &response); } } } @@ -322,22 +328,6 @@ impl SwitchyardRuntime { call.respond(result) } - async fn fallback_buffered( - &self, - inbound: WireFormat, - request: Request, - marks: &mut Vec, - metadata: &Json, - ) -> Result { - let response = self - .fallback_response(inbound, request, marks, metadata) - .await?; - let LlmResponse::Agg(response) = response.llm_response else { - return Err("trusted fallback returned a stream for a buffered request".into()); - }; - translation::encode_response(&self.translation, inbound, &response) - } - async fn fallback_response( &self, inbound: WireFormat, From f035e0de4448feb809e4ce8157533aae29c796a9 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 20:08:49 -0600 Subject: [PATCH 07/15] docs(relay): keep plugin scope isolated Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 8 -------- crates/switchyard-nemo-relay-plugin/README.md | 19 +++++-------------- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cadfdbc0..dab2e96ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,14 +14,6 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and fallback. Managed calls require NeMo Relay 0.7 or newer and do not depend on `switchyard-server`. -### Changed - -- **Switchyard HTTP transport limits** — provider redirects are rejected, - connection and read-inactivity timeouts are enforced, buffered success and - error bodies are bounded, and oversized SSE events fail before unbounded line - buffering. Provider error bodies remain available in typed errors but are no - longer included in their default display text. - ### Removed - **Latency-aware router** — the `latency_service` route type and its diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 9de160c6e..e5e1486b8 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -87,10 +87,9 @@ cancellation. A buffered callback has no cancellation token, so a provider request already in progress continues until it responds or reaches the client timeout after the caller disconnects. Relay can cancel a streaming iterator between pulls, but its synchronous `Iterator::next` call cannot be interrupted -while it is waiting for the next provider event. The configured 120-second -inactivity timeout bounds both cases. Supporting prompt disconnect propagation -would require an asynchronous public SDK callback or stream-polling contract; -the plugin does not bypass the SDK to recover that behavior. +while it is waiting for the next provider event. Supporting prompt disconnect +propagation would require an asynchronous public SDK callback or stream-polling +contract; the plugin does not bypass the SDK to recover that behavior. ### Relay runtime capacity @@ -131,9 +130,8 @@ let runtime = tokio::runtime::Builder::new_multi_thread() The durable resolution is a public Relay SDK callback and stream-polling contract that can yield while provider I/O is pending and receive caller -cancellation. Until that exists, keep the provider timeout bounded, provision -worker capacity, and load-test the intended concurrency rather than relying on -thread-pool growth alone. +cancellation. Until that exists, provision worker capacity and load-test the +intended concurrency rather than relying on thread-pool growth alone. Unmanaged profiles call the typed `LlmNext` or `LlmStreamNext` continuation and return its result directly. Managed streams use the bounded Switchyard channel; @@ -234,13 +232,6 @@ Common credential headers, including `authorization` and `x-api-key`, are rejected in static `headers` and must use `header_env`. Static headers remain appropriate for non-secret routing or tenancy metadata. -The Switchyard client does not follow provider redirects, applies a 10-second -connect timeout and a 120-second inactivity timeout, caps buffered provider -responses at 64 MiB, and caps retained HTTP error bodies at 64 KiB. Default -error display and Relay rejection messages do not include provider bodies. The -shared SSE decoder rejects an individual provider event above 8 MiB before its -line buffer can grow without bound. - For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or `openai_responses`; libsy's judge request uses a JSON-schema response format that cannot be represented losslessly by Anthropic Messages. The remaining From bd4526264a080f6294f3fc9d32cbda3ce2bda513 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 20:52:26 -0600 Subject: [PATCH 08/15] fix(relay): prevent repeated stream fallback Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index bb86751e8..b16d0048a 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -146,7 +146,7 @@ impl SwitchyardRuntime { json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); - let (response, fallback_used) = match self + let (response, mut fallback_used) = match self .drive(request.clone(), attempt, &mut marks, &metadata) .await { @@ -202,6 +202,7 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + fallback_used = true; let fallback = self .fallback_response(inbound, request.clone(), &mut marks, &metadata) .await?; From 93a8bbd8a284446168adbbb6cbf145bc008869a5 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 21:02:27 -0600 Subject: [PATCH 09/15] fix(relay): reject empty translated streams Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/src/runtime.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index b16d0048a..9fddbff05 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -139,7 +139,7 @@ impl SwitchyardRuntime { let max_attempts = self.max_retries + 1; let mut attempt = 1; let mut marks = Vec::new(); - loop { + 'attempts: loop { self.mark( &mut marks, "switchyard.routing.requested", @@ -237,7 +237,7 @@ impl SwitchyardRuntime { ); attempt += 1; send_marks(output, &mut marks).await?; - break; + continue 'attempts; } Err(failure) if !fallback_used && !committed => { self.mark( @@ -283,9 +283,7 @@ impl SwitchyardRuntime { if committed { return Ok(()); } - // A retry path breaks the event loop before commitment and starts a - // fresh libsy run. A successfully encoded stream always emits at - // least one event because `returned_events` rejects empty inputs. + return Err("Switchyard response stream produced no caller events".into()); } } From 81bd1ee8d498ae935a6e1f4cbc947fefb7335c3c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 21:37:46 -0600 Subject: [PATCH 10/15] fix(relay): back off routing retries Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 6 ++-- .../src/runtime.rs | 34 +++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index e5e1486b8..e218b1d05 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -157,8 +157,10 @@ Each target's built-in HTTP retry count is set to zero to avoid retrying a failed target invisibly before reselection. A random target with `weight = 0` is fallback-only and is not considered by the algorithm. Trusted fallback is attempted at most once and, for streaming responses, only before the first -caller event is emitted. Outer routing retries are immediate and do not honor -provider `Retry-After` headers. +caller event is emitted. Outer routing retries use exponential backoff starting +at 250 milliseconds and capped at 2 seconds. They do not currently honor +provider `Retry-After` headers because the client error contract does not expose +that metadata to the routing loop. ## Translation and stream fidelity diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 9fddbff05..96e8b8340 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use std::sync::Arc; +use std::time::Duration; use futures_util::{StreamExt, stream}; use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; @@ -17,6 +18,9 @@ use switchyard_translation::{TranslationEngine, encode_stream}; use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; use crate::translation; +const INITIAL_RETRY_BACKOFF: Duration = Duration::from_millis(250); +const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2); + #[derive(Debug)] pub(crate) struct RoutingMark { pub(crate) name: String, @@ -106,6 +110,7 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); + sleep_before_retry(attempt).await; attempt += 1; } Err(failure) => { @@ -158,8 +163,9 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); - attempt += 1; send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; continue; } Err(failure) => { @@ -191,8 +197,9 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); - attempt += 1; send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; continue; } Err(failure) if !fallback_used => { @@ -235,8 +242,9 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); - attempt += 1; send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; continue 'attempts; } Err(failure) if !fallback_used && !committed => { @@ -471,6 +479,17 @@ fn libsy_error_retryable(error: &LibsyError) -> bool { } } +fn retry_backoff(attempt: u32) -> Duration { + let exponent = attempt.saturating_sub(1).min(3); + INITIAL_RETRY_BACKOFF + .saturating_mul(1_u32 << exponent) + .min(MAX_RETRY_BACKOFF) +} + +async fn sleep_before_retry(attempt: u32) { + tokio::time::sleep(retry_backoff(attempt)).await; +} + fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { let mut data = Map::from_iter([ ("attempt".into(), Json::from(attempt)), @@ -595,6 +614,15 @@ fn context_from_metadata(metadata: Option<&Metadata>) -> Context { mod tests { use super::*; + #[test] + fn retry_backoff_increases_exponentially_and_is_capped() { + assert_eq!(retry_backoff(1), Duration::from_millis(250)); + assert_eq!(retry_backoff(2), Duration::from_millis(500)); + assert_eq!(retry_backoff(3), Duration::from_secs(1)); + assert_eq!(retry_backoff(4), Duration::from_secs(2)); + assert_eq!(retry_backoff(u32::MAX), Duration::from_secs(2)); + } + #[test] fn context_carries_identity_without_http_headers() { let context = context_from_metadata(Some(&Metadata { From 4844e8d53c032d7c952ea7e25e36f298235e33c3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 22:22:23 -0600 Subject: [PATCH 11/15] test(relay): cover single stream fallback Signed-off-by: Bryan Bednarski --- .../src/runtime.rs | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 96e8b8340..6798a37c2 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -612,8 +612,93 @@ fn context_from_metadata(metadata: Option<&Metadata>) -> Context { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use switchyard_libsy::{LlmTarget, Passthrough}; + use switchyard_protocol::{LlmResponseStream, RoutedLlmClient}; + use super::*; + #[derive(Clone, Copy)] + enum StreamBehavior { + Empty, + Failing, + } + + struct StreamClient { + behavior: StreamBehavior, + calls: AtomicUsize, + } + + #[async_trait::async_trait] + impl RoutedLlmClient for StreamClient { + async fn call( + &self, + _ctx: Context, + _request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + let stream: LlmResponseStream = match self.behavior { + StreamBehavior::Empty => Box::pin(stream::empty()), + StreamBehavior::Failing => Box::pin(stream::once(async { + Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback stream failed")), + }) + })), + }; + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } + } + + #[tokio::test] + async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(StreamClient { + behavior: StreamBehavior::Failing, + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + llm_client: Some(selected.clone()), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let (output, _messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback stream must fail the request"); + + assert_eq!(error, "trusted fallback stream: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + } + #[test] fn retry_backoff_increases_exponentially_and_is_capped() { assert_eq!(retry_backoff(1), Duration::from_millis(250)); From 49271a0b3df110ce53e3abddf74224105f5e493b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 23:22:00 -0600 Subject: [PATCH 12/15] fix(relay): harden response finalization Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 18 +- .../src/runtime.rs | 192 +++++++++++++++--- 2 files changed, 174 insertions(+), 36 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index e218b1d05..0ee87d3a8 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -169,15 +169,15 @@ layer. It decodes caller JSON into Switchyard's neutral protocol, encodes each selected call for the target protocol, decodes provider results, and encodes `ReturnToAgent` back to the caller protocol. Relay codecs are not used. -The current streaming contract uses the normalized `LlmResponseChunk` -representation and does not preserve a raw provider-event envelope. -Common text, usage, finish-reason, and tool-call fields can be translated, but -unknown provider-specific fields in same-protocol SSE events are not guaranteed -to survive the decode/libsy/encode round trip. The streaming helpers also do not -expose the buffered translation engine's reject-lossy diagnostics, so unsupported -cross-protocol stream fields may be normalized or omitted. Do not claim -lossless streaming until Switchyard exposes both a raw provider-event -preservation contract and an explicit reject-lossy stream policy. +The streaming contract carries each parsed provider JSON event in a preservation +envelope alongside its normalized `LlmResponseChunk` representation. +Same-protocol routes replay the preserved JSON unchanged, including +provider-specific fields; this preserves parsed events, not raw SSE bytes or +framing. Cross-protocol routes encode only normalized chunks, and the streaming +helpers still do not expose the buffered translation engine's reject-lossy +diagnostics, so unsupported fields may be normalized or omitted. Replacing +normalized stream content or folding a stream into an aggregate drops the +per-event preservation envelope. ## Configuration diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 6798a37c2..bdd6f312f 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -96,13 +96,15 @@ impl SwitchyardRuntime { json!({"algorithm": self.algorithm.name(), "attempt": attempt}), &metadata, ); - match self.drive(request.clone(), attempt, marks, &metadata).await { - Ok(response) => { - let LlmResponse::Agg(response) = response.llm_response else { - return Err("libsy returned a stream for a buffered request".into()); - }; - return translation::encode_response(&self.translation, inbound, &response); - } + let result = self + .drive(request.clone(), attempt, marks, &metadata) + .await + .and_then(|response| { + finalize_buffered_response(&self.translation, inbound, response) + .map_err(|source| LibsyError::client_call("return_to_agent", source)) + }); + match result { + Ok(response) => return Ok(response), Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { self.mark( marks, @@ -123,12 +125,10 @@ impl SwitchyardRuntime { let response = self .fallback_response(inbound, request, marks, &metadata) .await?; - let LlmResponse::Agg(response) = response.llm_response else { - return Err( - "trusted fallback returned a stream for a buffered request".into() - ); - }; - return translation::encode_response(&self.translation, inbound, &response); + return finalize_buffered_response(&self.translation, inbound, response) + .map_err(|error| { + public_response_failure("trusted fallback response", &error) + }); } } } @@ -432,6 +432,22 @@ async fn send_event( type ReturnedEventStream = std::pin::Pin> + Send>>; +fn finalize_buffered_response( + translation_engine: &TranslationEngine, + inbound: WireFormat, + response: Response, +) -> Result { + let LlmResponse::Agg(response) = response.llm_response else { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::other( + "libsy returned a stream for a buffered request", + )), + }); + }; + translation::encode_response(translation_engine, inbound, &response) + .map_err(LlmClientError::ResponseTranslation) +} + async fn returned_events( response: Response, inbound: WireFormat, @@ -545,6 +561,16 @@ fn public_libsy_failure(prefix: &str, error: &LibsyError) -> String { } } +fn public_response_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::InvalidResponse { .. } => format!("{prefix}: invalid response"), + LlmClientError::ResponseTranslation(_) => { + format!("{prefix}: response translation failure") + } + _ => format!("{prefix}: response finalization failure"), + } +} + fn public_client_failure(prefix: &str, error: &LlmClientError) -> String { match error { LlmClientError::UpstreamHttp { status, .. } => { @@ -630,6 +656,10 @@ mod tests { calls: AtomicUsize, } + struct BufferedClient { + calls: AtomicUsize, + } + #[async_trait::async_trait] impl RoutedLlmClient for StreamClient { async fn call( @@ -654,6 +684,122 @@ mod tests { } } + #[async_trait::async_trait] + impl RoutedLlmClient for BufferedClient { + async fn call( + &self, + _ctx: Context, + _request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok(Response { + llm_response: LlmResponse::Agg(Default::default()), + metadata: None, + }) + } + } + + #[tokio::test] + async fn buffered_finalization_failure_uses_fallback_once() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(BufferedClient { + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 1, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + llm_client: Some(selected.clone()), + })), + targets: BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + )]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) + .await + .expect("the buffered fallback response should be encoded"); + + assert!(response.is_object()); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.retry") + ); + let error = marks + .iter() + .find(|mark| mark.name == "switchyard.routing.error") + .expect("finalization failure should emit an error mark"); + assert_eq!(error.data["retryable"], false); + assert_eq!(error.data["non_http_kind"], "invalid_response"); + assert_eq!( + marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.fallback") + .count(), + 1 + ); + } + + #[tokio::test] + async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { + let content = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }); + let terminal = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop" + }] + }); + let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); + let stream = switchyard_translation::decode_stream( + stream::once(async move { Ok::<_, LlmClientError>(body) }), + WireFormat::OpenAiChat, + ) + .expect("provider SSE should decode"); + let response = Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }; + + let replayed = returned_events(response, WireFormat::OpenAiChat) + .await + .expect("return stream should encode") + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("return stream should not fail"); + + assert_eq!(replayed, vec![content, terminal]); + } + #[tokio::test] async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { let selected = Arc::new(StreamClient { @@ -670,20 +816,12 @@ mod tests { semantic_name: "selected".into(), llm_client: Some(selected.clone()), })), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), + targets: BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + )]), default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), translation: TranslationEngine::default(), }; From 504f29d4145be750276f76401758547119485b8b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 3 Aug 2026 23:38:47 -0600 Subject: [PATCH 13/15] chore(relay): update SDK to 0.7.0-rc.5 Signed-off-by: Bryan Bednarski --- Cargo.lock | 98 +------------------ .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 4 +- .../relay-plugin.toml | 2 +- 4 files changed, 8 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa5e11a2b..8d9f415f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,15 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "1.0.0" @@ -317,12 +308,8 @@ version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ - "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", - "windows-link", ] [[package]] @@ -831,30 +818,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iana-time-zone" -version = "0.1.65" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "icu_collections" version = "2.2.0" @@ -1150,9 +1113,9 @@ dependencies = [ [[package]] name = "nemo-relay-plugin" -version = "0.7.0-rc.4" +version = "0.7.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ac90070f6fe5778d882b7c0a1457441a698a9d6ab85e9514e9a3fefed87dbc" +checksum = "ae9f5d2e2f16f0db4e9b666068fb3c470eb655b6067ac7f5fa4c66ce8051c220" dependencies = [ "nemo-relay-types", "serde", @@ -1161,9 +1124,9 @@ dependencies = [ [[package]] name = "nemo-relay-types" -version = "0.7.0-rc.4" +version = "0.7.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a1a46140ebab2df2201d385aad95e62149bd1d4a230efe74df7ec7b594cad24" +checksum = "70b2358889074b00eba89ec42bc8114f3515541f928306ed770ddf995cfa416b" dependencies = [ "bitflags", "chrono", @@ -2821,65 +2784,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 376778bdb..4806da86f 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -22,7 +22,7 @@ futures-util.workspace = true http = "1" # Use the published release candidate during development. This becomes `0.7.0` # before the integration lands, after the stable crate is published. -nemo-relay-plugin = "=0.7.0-rc.4" +nemo-relay-plugin = "=0.7.0-rc.5" serde.workspace = true serde_json.workspace = true switchyard-libsy.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 0ee87d3a8..e59e776a8 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -182,8 +182,8 @@ per-event preservation envelope. ## Configuration During release-candidate validation the manifest declares -`compat.native_api = "1"` and Relay `>=0.7.0-rc.4,<1.0`, and the Rust SDK uses -the exact published `0.7.0-rc.4` crate. Before release, move both lower bounds +`compat.native_api = "1"` and Relay `>=0.7.0-rc.5,<1.0`, and the Rust SDK uses +the exact published `0.7.0-rc.5` crate. Before release, move both lower bounds to stable `0.7.0`. The manifest API value selects Relay's released native plugin contract; plugin authors use its safe Rust SDK rather than the underlying C table directly. Rebuild the bundle when changing SDK versions rather than diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml index 9ba7c2bfc..0d7a75f12 100644 --- a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -10,7 +10,7 @@ kind = "rust_dynamic" [compat] # Development lower bound for validation against the published Relay RC. Move # this to `>=0.7.0,<1.0` together with the SDK dependency before release. -relay = ">=0.7.0-rc.4,<1.0" +relay = ">=0.7.0-rc.5,<1.0" native_api = "1" [defaults] From f63274d0ebc43a96078eb6c4048ee73544c35789 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 4 Aug 2026 12:14:31 -0600 Subject: [PATCH 14/15] fix(relay): flush marks before fallback errors Signed-off-by: Bryan Bednarski --- .../src/runtime.rs | 74 +++++++++++++++++-- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index bdd6f312f..8898ef6d3 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -175,11 +175,11 @@ impl SwitchyardRuntime { failure_mark_data(attempt, &failure), &metadata, ); - ( - self.fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await?, - true, - ) + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + (fallback?, true) } }; send_marks(output, &mut marks).await?; @@ -212,8 +212,9 @@ impl SwitchyardRuntime { fallback_used = true; let fallback = self .fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await?; + .await; send_marks(output, &mut marks).await?; + let fallback = fallback?; returned_events(fallback, inbound) .await .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? @@ -256,8 +257,9 @@ impl SwitchyardRuntime { ); let fallback = self .fallback_response(inbound, request.clone(), &mut marks, &metadata) - .await?; + .await; send_marks(output, &mut marks).await?; + let fallback = fallback?; let mut fallback = returned_events(fallback, inbound).await.map_err(|error| { public_libsy_failure("trusted fallback stream", &error) @@ -649,6 +651,7 @@ mod tests { enum StreamBehavior { Empty, Failing, + CallFailure, } struct StreamClient { @@ -676,6 +679,11 @@ mod tests { source: Box::new(std::io::Error::other("fallback stream failed")), }) })), + StreamBehavior::CallFailure => { + return Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback call failed")), + }); + } }; Ok(Response { llm_response: LlmResponse::Stream(stream), @@ -837,6 +845,58 @@ mod tests { assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); } + #[tokio::test] + async fn failing_fallback_call_flushes_error_and_fallback_marks() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(StreamClient { + behavior: StreamBehavior::CallFailure, + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + llm_client: Some(selected.clone()), + })), + targets: BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + )]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + translation: TranslationEngine::default(), + }; + let (output, messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback call must fail the request"); + + assert_eq!(error, "trusted fallback: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + let mut terminal_marks = Vec::new(); + while let Ok(message) = messages.try_recv() { + if let StreamMessage::Mark(mark) = message + && matches!( + mark.name.as_str(), + "switchyard.routing.error" | "switchyard.routing.fallback" + ) + { + terminal_marks.push(mark.name); + } + } + assert_eq!( + terminal_marks, + ["switchyard.routing.error", "switchyard.routing.fallback"] + ); + } + #[test] fn retry_backoff_increases_exponentially_and_is_capped() { assert_eq!(retry_backoff(1), Duration::from_millis(250)); From c69a8b68f7c85e4b610c077690f90db6de9053ed Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 4 Aug 2026 12:36:07 -0600 Subject: [PATCH 15/15] fix(relay): require environment-backed target headers Signed-off-by: Bryan Bednarski --- crates/switchyard-nemo-relay-plugin/README.md | 22 +++--- .../config.schema.json | 7 +- .../src/client.rs | 2 +- .../src/config.rs | 79 +++++++------------ 4 files changed, 43 insertions(+), 67 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index e59e776a8..53ee58e3c 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -222,17 +222,17 @@ authorization = "PROVIDER_AUTHORIZATION" Target map keys such as `fast` are stable semantic names visible to libsy. The target binding is authoritative for the provider model, protocol, endpoint, -base URL, weight, and headers. Each `default_targets` key both enables that -inbound protocol and names its trusted fallback. - -`header_env` resolves target credentials in the plugin process at registration -time. Environment values must not appear in configuration, errors, routing -marks, spans, or debug output. The plugin does not inherit caller credentials -for managed calls. Each variable supplies the complete header value, so an -`authorization` value must include its scheme, such as `Bearer`. -Common credential headers, including `authorization` and `x-api-key`, are -rejected in static `headers` and must use `header_env`. Static headers remain -appropriate for non-secret routing or tenancy metadata. +base URL, weight, and environment-backed headers. Each `default_targets` key +both enables that inbound protocol and names its trusted fallback. + +`header_env` is the only custom provider-header source. It resolves values in +the plugin process at registration time so literal header values never appear +in configuration. Environment values must not appear in errors, routing marks, +spans, or debug output. The plugin does not inherit caller credentials for +managed calls. Each variable supplies the complete header value, so an +`authorization` value must include its scheme, such as `Bearer`. Literal +`headers` configuration is rejected; non-secret routing or tenancy headers must +also use `header_env`. For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or `openai_responses`; libsy's judge request uses a JSON-schema response format diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json index 3ac510bfd..6305755a8 100644 --- a/crates/switchyard-nemo-relay-plugin/config.schema.json +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -96,14 +96,9 @@ "pattern": "^https?://" }, "weight": { "type": "number", "minimum": 0, "default": 1 }, - "headers": { - "type": "object", - "description": "Static non-secret provider headers. Credential-bearing headers must use header_env.", - "additionalProperties": { "type": "string" } - }, "header_env": { "type": "object", - "description": "Maps provider header names to environment-variable names resolved by the plugin process.", + "description": "Sole custom provider-header source. Maps header names to environment-variable names resolved by the plugin process so literal values are never stored in configuration.", "additionalProperties": { "type": "string", "minLength": 1 } } } diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs index ae15ab8fe..316a658da 100644 --- a/crates/switchyard-nemo-relay-plugin/src/client.rs +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -66,7 +66,7 @@ impl TargetClient { /// /// Correlation and agent identity remain available to libsy, while inbound /// HTTP headers are deliberately removed. Provider credentials come solely - /// from this target's `headers` / `header_env` configuration. + /// from this target's `header_env` configuration. fn prepare_request(&self, mut request: Request) -> Request { let metadata = request.metadata.get_or_insert_default(); metadata.wire_format = Some(self.target_format); diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 1e1538aa8..fa1ee4ec9 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -42,8 +42,6 @@ struct TargetBinding { #[serde(default = "default_weight")] weight: f64, #[serde(default)] - headers: BTreeMap, - #[serde(default)] header_env: BTreeMap, } @@ -87,24 +85,11 @@ impl TargetBinding { fn validate_headers(&self, target_name: &str) -> Result<(), String> { let mut normalized = BTreeSet::new(); - for (name, value) in &self.headers { - let canonical = validate_header(name, value)?; - if is_sensitive_target_header(&canonical) { - return Err(format!( - "target {target_name:?} header {name:?} must be supplied through header_env so its value is not stored in Relay configuration" - )); - } - if !normalized.insert(canonical) { - return Err(format!( - "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" - )); - } - } for (name, variable) in &self.header_env { let canonical = validate_header_name(name)?; if !normalized.insert(canonical) { return Err(format!( - "target {target_name:?} configures header {name:?} more than once across headers and header_env" + "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" )); } if variable.trim().is_empty() { @@ -122,7 +107,7 @@ impl TargetBinding { } fn prepare(&self) -> Result { - let mut headers = self.headers.clone(); + let mut headers = BTreeMap::new(); for (name, variable) in &self.header_env { let value = std::env::var(variable) .map_err(|_| format!("environment variable {variable:?} is not set"))?; @@ -399,18 +384,6 @@ fn is_forbidden_target_header(name: &str) -> bool { ) || name.starts_with("x-nemo-relay-internal-") } -fn is_sensitive_target_header(name: &str) -> bool { - matches!( - name, - "authorization" - | "cookie" - | "x-api-key" - | "api-key" - | "anthropic-api-key" - | "x-goog-api-key" - ) -} - const fn default_max_retries() -> u32 { 3 } @@ -431,7 +404,6 @@ mod tests { endpoint: String::new(), base_url: "https://provider.example/v1".into(), weight: 1.0, - headers: BTreeMap::new(), header_env: BTreeMap::new(), } } @@ -533,11 +505,10 @@ mod tests { } #[test] - fn transport_owned_and_case_duplicate_headers_are_rejected() { + fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { let mut host_header_config = config(); let chat = host_header_config.targets.get_mut("chat").unwrap(); - chat.headers - .insert("Host".into(), "attacker.example".into()); + chat.header_env.insert("Host".into(), "TARGET_HOST".into()); assert!( host_header_config .validate() @@ -545,25 +516,12 @@ mod tests { .contains("HTTP transport") ); - let mut static_secret_config = config(); - static_secret_config - .targets - .get_mut("chat") - .unwrap() - .headers - .insert("Authorization".into(), "Bearer target-secret".into()); - assert!( - static_secret_config - .validate() - .unwrap_err() - .contains("must be supplied through header_env") - ); - let mut duplicate_config = config(); let chat = duplicate_config.targets.get_mut("chat").unwrap(); - chat.headers.insert("X-Tenant".into(), "blue".into()); chat.header_env - .insert("x-tenant".into(), "TARGET_TENANT".into()); + .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); + chat.header_env + .insert("x-tenant".into(), "TARGET_TENANT_B".into()); assert!( duplicate_config .validate() @@ -634,6 +592,29 @@ mod tests { assert!(error.to_string().contains("unexpected_setting")); } + #[test] + fn literal_target_headers_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "headers": {"x-provider-token": "plaintext-secret"} + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("literal target headers must be rejected") + .to_string(); + assert!(error.contains("unknown field `headers`")); + assert!(!error.contains("plaintext-secret")); + } + #[test] fn unknown_algorithm_fields_are_rejected() { let error = serde_json::from_value::(json!({