fix: handle lone Unicode surrogates in artifact persistence

External tool outputs (e.g. Exa Search) can contain lone Unicode
surrogate characters (U+D800..U+DFFF) that are invalid in both UTF-8
and JSON, causing UnicodeEncodeError in _serialize_payload_bytes and
surfacing as "Tool storage unavailable" in the chat UI.

- Add _sanitize_surrogates() to recursively strip lone surrogates and
  null bytes from payload dicts before serialization, using the same
  encode-surrogatepass/decode-ignore technique as OWUI's
  sanitize_text_for_db
- Move _prepare_rows_for_storage inside the try-except in _db_persist
  so serialization errors return [] instead of crashing streaming
- Add 5 regression tests covering surrogates, null bytes, valid
  Unicode preservation, recursive sanitization, and round-trip safety

Author: rbb-dev

open_webui_openrouter_pipe/storage/persistence.py

@@ -687,9 +687,37 @@ def _should_encrypt(self, item_type: str) -> bool:
             return True
         return (item_type or "").lower() == "reasoning"
 
+    @staticmethod
+    def _sanitize_surrogates(obj: Any) -> Any:
+        """Recursively strip lone surrogates and null bytes from all strings.
+
+        Lone surrogates (U+D800..U+DFFF) are invalid in both UTF-8 and JSON,
+        and cause ``UnicodeEncodeError`` / ``json.dumps`` failures.  External
+        tool outputs (e.g. web-search results) may contain them when source
+        data has encoding issues.
+
+        Uses the same encode-surrogatepass/decode-ignore technique as
+        Open WebUI's ``sanitize_text_for_db`` (``open_webui.utils.misc``).
+        """
+        if isinstance(obj, str):
+            # Remove null bytes, then strip lone surrogates
+            cleaned = obj.replace("\x00", "")
+            try:
+                return cleaned.encode("utf-8", errors="surrogatepass").decode(
+                    "utf-8", errors="ignore"
+                )
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                return cleaned
+        if isinstance(obj, dict):
+            return {k: ArtifactStore._sanitize_surrogates(v) for k, v in obj.items()}
+        if isinstance(obj, list):
+            return [ArtifactStore._sanitize_surrogates(v) for v in obj]
+        return obj
+
     def _serialize_payload_bytes(self, payload: dict[str, Any]) -> bytes:
         """Return compact JSON bytes for ``payload``."""
-        return json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+        sanitized = self._sanitize_surrogates(payload)
+        return json.dumps(sanitized, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
 
     def _maybe_compress_payload(self, serialized: bytes) -> tuple[bytes, bool]:
         """Compress serialized bytes when LZ4 is available and thresholds are met."""
@@ -1018,9 +1046,8 @@ async def _db_persist(self, rows: list[dict[str, Any]]) -> list[str]:
         for row in rows:
             row.setdefault("id", generate_item_id())
 
-        self._prepare_rows_for_storage(rows)
-
         try:
+            self._prepare_rows_for_storage(rows)
             if self._redis_enabled:
                 return await self._redis_enqueue_rows(rows)
             return await self._db_persist_direct(rows, user_id=user_id)

tests/test_persistence.py

@@ -585,6 +585,68 @@ def test_prepare_rows_for_storage_non_dict_payload(pipe_instance):
     assert rows[1]["payload"] == 123
 
 
+def test_sanitize_surrogates_strips_lone_surrogates():
+    """_sanitize_surrogates removes lone surrogates that crash json.dumps/UTF-8 encode."""
+    from open_webui_openrouter_pipe.storage.persistence import ArtifactStore
+
+    # Lone high surrogate (the exact character from the Exa Search crash)
+    text_with_surrogate = f"Hello \ud835 world"
+    result = ArtifactStore._sanitize_surrogates(text_with_surrogate)
+    assert "\ud835" not in result
+    assert "Hello" in result and "world" in result
+    # Must be safe for JSON + UTF-8
+    import json
+    encoded = json.dumps(result, ensure_ascii=False).encode("utf-8")
+    assert isinstance(encoded, bytes)
+
+
+def test_sanitize_surrogates_preserves_valid_text():
+    """_sanitize_surrogates preserves normal Unicode text including emoji and CJK."""
+    from open_webui_openrouter_pipe.storage.persistence import ArtifactStore
+
+    valid = "Hello 🌍 world 你好 café"
+    assert ArtifactStore._sanitize_surrogates(valid) == valid
+
+
+def test_sanitize_surrogates_recursive():
+    """_sanitize_surrogates recursively cleans nested dicts and lists."""
+    from open_webui_openrouter_pipe.storage.persistence import ArtifactStore
+
+    payload = {
+        "output": f"result \ud835 text",
+        "nested": {"deep": [f"item \ud800"]},
+    }
+    result = ArtifactStore._sanitize_surrogates(payload)
+    assert "\ud835" not in result["output"]
+    assert "\ud800" not in result["nested"]["deep"][0]
+    # Non-string values pass through
+    assert ArtifactStore._sanitize_surrogates(42) == 42
+    assert ArtifactStore._sanitize_surrogates(None) is None
+
+
+def test_serialize_payload_bytes_with_surrogates(pipe_instance):
+    """_serialize_payload_bytes handles payloads containing lone surrogates."""
+    store = pipe_instance._artifact_store
+    payload = {"output": f"math \ud835 symbols", "type": "function_call_output"}
+    result = store._serialize_payload_bytes(payload)
+    assert isinstance(result, bytes)
+    # Must round-trip through JSON
+    import json
+    parsed = json.loads(result.decode("utf-8"))
+    assert "math" in parsed["output"]
+    assert "\ud835" not in parsed["output"]
+
+
+def test_sanitize_surrogates_strips_null_bytes():
+    """_sanitize_surrogates also removes null bytes (PostgreSQL incompatible)."""
+    from open_webui_openrouter_pipe.storage.persistence import ArtifactStore
+
+    text = "before\x00after"
+    result = ArtifactStore._sanitize_surrogates(text)
+    assert "\x00" not in result
+    assert result == "beforeafter"
+
+
 # -----------------------------------------------------------------------------
 # Tests for DB Persist Edge Cases (lines 812, 916, 975, 980)
 # -----------------------------------------------------------------------------