From f72d394b9e722ff0f13fb3501aedf295aea5025a Mon Sep 17 00:00:00 2001 From: Matheus Pastorini Date: Mon, 27 Jul 2026 01:23:50 -0300 Subject: [PATCH 1/2] feat(runners): transcribe audio with the agent's own multimodal model (EVO-2227 Fase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Voice notes reached the agent but 500'd at the LLM: google-adk 1.19.0 converts an audio Blob into an `audio_url` content part (lite_llm.py `_get_content`), and litellm 1.68.2 rejects `audio_url` — it is not in ValidUserMessageContentTypes (text/image_url/input_audio/document/video_url/file). The whole turn was lost, the user's caption included. Audio is no longer forwarded as a raw model part. It is transcribed with the agent's OWN configured model and the transcript stands in as text, so ANY answering LLM understands the voice note. Images still inline natively via image_url (EVO-2181), untouched. Provider routing (both ingest WhatsApp opus/ogg without transcoding): - OpenAI family -> litellm.atranscription (whisper-1), the dedicated STT endpoint. - Gemini / other chat-audio -> litellm.acompletion with an input_audio part; "audio/opus" (codec label) is mapped to "audio/ogg" (container) so Gemini's accepted-MIME set recognizes it. Best-effort by contract: a None transcript (unsupported provider, network, quota, undecodable bytes) leaves the turn on its remaining text — never a 500. - New src/services/adk/runners/audio_transcription.py (transcribe_audio_file / transcribe_audio; reuses get_api_key + normalize_model_for_provider so it follows the agent's provider/key exactly, OpenRouter prefix included). - process_files: audio -> transcribed_texts (folded into the message by the runner via create_content_with_transcribed_audio), no inline part. - Tests: 15 for the transcription service (routing, opus->ogg, best-effort, db/agent guards) + updated media-parts contract. The media-parts suite now also runs litellm's validator on the forwarded parts and pins that an inlined audio part WOULD be rejected — the ADK-only check missed that and gave a false green. Full tests/unit: 265 -> 266+ green. --- .../adk/runners/audio_transcription.py | 204 ++++++++++++++++ src/services/adk/runners/runner_utils.py | 74 ++++-- tests/unit/test_audio_transcription.py | 220 ++++++++++++++++++ tests/unit/test_media_file_parts.py | 87 +++++-- 4 files changed, 543 insertions(+), 42 deletions(-) create mode 100644 src/services/adk/runners/audio_transcription.py create mode 100644 tests/unit/test_audio_transcription.py diff --git a/src/services/adk/runners/audio_transcription.py b/src/services/adk/runners/audio_transcription.py new file mode 100644 index 0000000..6ac6308 --- /dev/null +++ b/src/services/adk/runners/audio_transcription.py @@ -0,0 +1,204 @@ +"""EVO-2227 (Fase 2): transcribe an incoming audio attachment with the agent's +own multimodal model, so the transcript can be folded into the message text and +understood by ANY answering LLM. + +Why not inline the raw audio to the agent instead? google-adk 1.19.0 emits an +``audio_url`` content part for audio (``lite_llm.py`` ``_get_content``), and +litellm 1.68.2 rejects ``audio_url`` -- it is not in +``ValidUserMessageContentTypes`` (``text``/``image_url``/``input_audio``/ +``document``/``video_url``/``file``). The result is a 500 that costs the whole +turn. So audio never travels as a raw model part; it is transcribed here and the +text takes its place. Images still inline natively via ``image_url``. + +Provider routing (both accept WhatsApp opus/ogg without transcoding): +- OpenAI family -> ``litellm.atranscription`` (whisper-1), a dedicated STT + endpoint that ingests opus/ogg/m4a/mp3/wav/webm directly. +- Everything else that can hear audio in chat (Gemini) -> ``litellm.acompletion`` + with an ``input_audio`` part; litellm's Gemini transform consumes it + (``vertex_ai/gemini/transformation.py``). + +Best-effort by contract: any failure (unsupported provider, network, quota) +returns ``None`` so the turn survives on whatever text it already carries. +""" + +from __future__ import annotations + +import base64 +import io +from typing import Optional + +import litellm +from sqlalchemy.orm import Session + +from src.models.models import Agent +from src.services.adk.agents.agent_utils import get_api_key +from src.services.agent_service import get_agent +from src.utils.llm_model_routing import normalize_model_for_provider +from src.utils.logger import setup_logger + +logger = setup_logger(__name__) + +# Kept short and directive so the model returns just the words, no preamble. +_TRANSCRIBE_PROMPT = ( + "Transcribe the following audio verbatim. Return only the transcription " + "text in the audio's own language, with no extra commentary." +) + +# Model families whose keys route through OpenAI's STT endpoint. Matched against +# the lowercased model identifier. Anything else is tried via chat input_audio. +_OPENAI_MODEL_PREFIXES = ("gpt", "o1", "o3", "o4", "chatgpt", "whisper") + +# OpenAI's dedicated speech-to-text model. Reachable with the same key the agent +# already uses; ingests opus/ogg directly (unlike chat input_audio, which is +# wav/mp3 only). +_OPENAI_TRANSCRIBE_MODEL = "whisper-1" + +# WhatsApp voice notes arrive labeled "audio/opus", but the bytes are an OGG +# container (Opus codec) and Gemini's accepted audio MIME set lists "audio/ogg", +# not "audio/opus". Map the codec label to the container so the chat provider +# recognizes it. Only used on the chat (input_audio) path; whisper reads the file +# regardless of the label. +_CHAT_AUDIO_MIME_ALIASES = { + "audio/opus": "audio/ogg", + "audio/x-opus": "audio/ogg", +} + + +def _is_openai_family(model: str, provider: Optional[str]) -> bool: + """Whether to route transcription through OpenAI's STT endpoint. + + OpenRouter keys never hit OpenAI's STT endpoint directly, so they fall to the + chat path regardless of the underlying vendor. + """ + if provider == "openrouter": + return False + if provider == "openai": + return True + model_l = (model or "").lower() + if model_l.startswith("openai/"): + return True + return model_l.startswith(_OPENAI_MODEL_PREFIXES) + + +async def transcribe_audio_file( + db: Optional[Session], + agent_id: str, + content_type: str, + filename: str, + data_b64: str, +) -> Optional[str]: + """Resolve the agent's model/key and transcribe the base64 audio. + + Returns the transcript text, or ``None`` when transcription is impossible or + fails (the caller keeps the turn either way). + """ + if db is None or not agent_id: + return None + try: + agent = await get_agent(db, agent_id) + except Exception as e: # get_agent can raise on a bad id / db hiccup + logger.warning(f"[AudioTranscription] could not load agent {agent_id}: {e}") + return None + if agent is None: + return None + return await transcribe_audio(db, agent, content_type, filename, data_b64) + + +async def transcribe_audio( + db: Session, agent: Agent, content_type: str, filename: str, data_b64: str +) -> Optional[str]: + """Transcribe one audio attachment with the agent's configured model.""" + try: + raw_bytes = base64.b64decode(data_b64) + except Exception as e: + logger.warning(f"[AudioTranscription] undecodable audio {filename}: {e}") + return None + if not raw_bytes: + return None + + try: + api_key, provider = await get_api_key(db, agent) + except Exception as e: + logger.warning( + f"[AudioTranscription] no usable API key for agent {agent.id}: {e}" + ) + return None + + model = agent.model or "" + try: + if _is_openai_family(model, provider): + text = await _transcribe_via_openai( + api_key, content_type, filename, raw_bytes + ) + else: + text = await _transcribe_via_chat( + model, provider, api_key, content_type, raw_bytes + ) + except Exception as e: + logger.warning( + f"[AudioTranscription] transcription failed for {filename}" + f" (model={model!r}, provider={provider!r}): {e}" + ) + return None + + text = (text or "").strip() + if not text: + logger.info(f"[AudioTranscription] empty transcript for {filename}") + return None + logger.info( + f"[AudioTranscription] transcribed {filename} ({len(raw_bytes)} bytes)" + f" -> {len(text)} chars" + ) + return text + + +async def _transcribe_via_openai( + api_key: str, content_type: str, filename: str, raw_bytes: bytes +) -> Optional[str]: + """OpenAI STT (whisper-1). Ingests opus/ogg directly.""" + audio = io.BytesIO(raw_bytes) + # The SDK derives the format from the file name's extension; keep the real + # one so an .ogg/.opus is not mistaken for something the endpoint rejects. + audio.name = filename or "audio.ogg" + resp = await litellm.atranscription( + model=_OPENAI_TRANSCRIBE_MODEL, + file=audio, + api_key=api_key, + ) + # litellm returns a TranscriptionResponse with a .text attribute. + return getattr(resp, "text", None) + + +async def _transcribe_via_chat( + model: str, + provider: Optional[str], + api_key: str, + content_type: str, + raw_bytes: bytes, +) -> Optional[str]: + """Chat completion with an ``input_audio`` part (Gemini et al.).""" + norm_model, extra_kwargs = normalize_model_for_provider(model, provider) + # Normalize "audio/webm;codecs=opus" -> "audio/webm" for the data-uri header, + # then map codec labels (audio/opus) to the container MIME the provider knows. + mime = (content_type or "audio/ogg").split(";")[0].strip().lower() or "audio/ogg" + mime = _CHAT_AUDIO_MIME_ALIASES.get(mime, mime) + b64 = base64.b64encode(raw_bytes).decode("utf-8") + data_uri = f"data:{mime};base64,{b64}" + resp = await litellm.acompletion( + model=norm_model, + api_key=api_key, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": _TRANSCRIBE_PROMPT}, + { + "type": "input_audio", + "input_audio": {"data": data_uri, "format": mime}, + }, + ], + } + ], + **extra_kwargs, + ) + return resp.choices[0].message.content diff --git a/src/services/adk/runners/runner_utils.py b/src/services/adk/runners/runner_utils.py index f354f8c..1110015 100644 --- a/src/services/adk/runners/runner_utils.py +++ b/src/services/adk/runners/runner_utils.py @@ -39,6 +39,7 @@ from src.core.exceptions import AgentNotFoundError from src.services.agent_service import get_agent from src.services.adk.agent_builder import AgentBuilder +from src.services.adk.runners.audio_transcription import transcribe_audio_file from src.utils.adk_utils import extract_state_params from sqlalchemy.orm import Session from typing import Optional, List, Tuple, Dict, Any, Union @@ -294,31 +295,60 @@ async def process_files( ) ) - # EVO-2181: add the file to the content parts so the model - # actually receives it. This append used to be gated behind - # `if is_audio`, so an image was blobbed and saved to the - # artifact store but never handed to the LLM -> the agent - # replied "No content to process". - # - # It runs before save_artifact on purpose: the bytes are - # already in hand, and a failure while storing them must not - # cost the model its copy of the file and bring this very bug - # back. - skip_reason = self._inline_skip_reason( - file_data.content_type, len(file_bytes), inlined_bytes - ) - if skip_reason: - logger.warning( - f"File {file_data.filename} ({file_data.content_type}) not sent" - f" to the model: {skip_reason}. Still saved as an artifact." + if is_audio: + # EVO-2227 (Fase 2): audio never travels as a raw model + # part. google-adk 1.19.0 emits an `audio_url` content + # part for audio and litellm 1.68.2 rejects it (not in + # ValidUserMessageContentTypes) -> a 500 that costs the + # whole turn. Instead transcribe it with the agent's own + # multimodal model and let the text stand in, so ANY + # answering LLM understands the voice note. Best-effort: + # a None transcript leaves the turn on its remaining text. + transcript = await transcribe_audio_file( + getattr(self, "db", None), + agent_id, + file_data.content_type, + file_data.filename, + file_data.data, ) + if transcript: + transcribed_texts.append(transcript) + logger.info( + f"Transcribed audio {file_data.filename}" + f" ({file_data.content_type}); text stands in for the file" + ) + else: + logger.warning( + f"Audio {file_data.filename} could not be transcribed;" + f" the turn continues on its remaining text" + ) else: - inlined_bytes += len(file_bytes) - file_parts.append(file_part) - logger.info( - f"Added {'audio' if is_audio else 'file'} {file_data.filename}" - f" ({file_data.content_type}) to content parts for LLM processing" + # EVO-2181: add readable non-audio media (image/pdf/text/ + # video) to the content parts so the model actually + # receives it. This append used to be gated behind + # `if is_audio`, so an image was blobbed and saved to the + # artifact store but never handed to the LLM -> the agent + # replied "No content to process". + # + # It runs before save_artifact on purpose: the bytes are + # already in hand, and a failure while storing them must + # not cost the model its copy of the file and bring this + # very bug back. + skip_reason = self._inline_skip_reason( + file_data.content_type, len(file_bytes), inlined_bytes ) + if skip_reason: + logger.warning( + f"File {file_data.filename} ({file_data.content_type}) not sent" + f" to the model: {skip_reason}. Still saved as an artifact." + ) + else: + inlined_bytes += len(file_bytes) + file_parts.append(file_part) + logger.info( + f"Added file {file_data.filename}" + f" ({file_data.content_type}) to content parts for LLM processing" + ) # Always save to artifacts for reference await artifacts_service.save_artifact( diff --git a/tests/unit/test_audio_transcription.py b/tests/unit/test_audio_transcription.py new file mode 100644 index 0000000..f448f10 --- /dev/null +++ b/tests/unit/test_audio_transcription.py @@ -0,0 +1,220 @@ +"""EVO-2227 (Fase 2): transcribe audio with the agent's own multimodal model. + +Covers the provider routing (OpenAI STT vs chat input_audio), the best-effort +contract (any failure -> None so the turn survives), and the input shaping. +""" + +from __future__ import annotations + +import asyncio +import base64 +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import src.services.adk.runners.audio_transcription as at + +B64 = base64.b64encode(b"opus-bytes").decode() + + +def _run(coro): + return asyncio.run(coro) + + +def _agent(model="gemini/gemini-2.0-flash", agent_id="agent-1"): + return SimpleNamespace(id=agent_id, model=model, api_key_id=None, config={}) + + +# ---- _is_openai_family ------------------------------------------------------ + + +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("gpt-4o", None, True), + ("gpt-4.1-mini", None, True), + ("o3-mini", None, True), + ("openai/gpt-4o", None, True), + ("anything", "openai", True), + ("gemini/gemini-2.0-flash", None, False), + ("gemini-1.5-pro", None, False), + ("gpt-4o", "openrouter", False), # openrouter never hits OpenAI STT + ("claude-3-5-sonnet", None, False), + ], +) +def test_is_openai_family(model, provider, expected): + assert at._is_openai_family(model, provider) is expected + + +# ---- routing ---------------------------------------------------------------- + + +def test_openai_family_routes_to_whisper(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("sk-x", "openai"))) + atranscription = AsyncMock(return_value=SimpleNamespace(text="olá do whisper")) + monkeypatch.setattr(at.litellm, "atranscription", atranscription) + monkeypatch.setattr( + at.litellm, + "acompletion", + AsyncMock(side_effect=AssertionError("should not be called")), + ) + + text = _run( + at.transcribe_audio( + MagicMock(), _agent(model="gpt-4o"), "audio/ogg", "v.ogg", B64 + ) + ) + + assert text == "olá do whisper" + assert atranscription.await_count == 1 + kwargs = atranscription.await_args.kwargs + assert kwargs["model"] == "whisper-1" + assert kwargs["api_key"] == "sk-x" + + +def test_gemini_routes_to_chat_input_audio(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="olá do gemini"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + monkeypatch.setattr( + at.litellm, + "atranscription", + AsyncMock(side_effect=AssertionError("should not be called")), + ) + + text = _run( + at.transcribe_audio( + MagicMock(), + _agent(model="gemini/gemini-2.0-flash"), + "audio/webm;codecs=opus", + "v.webm", + B64, + ) + ) + + assert text == "olá do gemini" + msg = acompletion.await_args.kwargs["messages"][0] + part = msg["content"][1] + assert part["type"] == "input_audio" + # mime parameter stripped for the data-uri header + assert part["input_audio"]["data"].startswith("data:audio/webm;base64,") + assert part["input_audio"]["format"] == "audio/webm" + + +def test_opus_label_is_mapped_to_ogg_container_for_chat(monkeypatch): + # WhatsApp sends "audio/opus" but the bytes are an OGG container; Gemini + # knows "audio/ogg", not "audio/opus". + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/opus", "v.ogg", B64)) + + part = acompletion.await_args.kwargs["messages"][0]["content"][1] + assert part["input_audio"]["data"].startswith("data:audio/ogg;base64,") + assert part["input_audio"]["format"] == "audio/ogg" + + +def test_openrouter_uses_chat_path_with_api_base(monkeypatch): + monkeypatch.setattr( + at, "get_api_key", AsyncMock(return_value=("or-key", "openrouter")) + ) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="via openrouter"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + + text = _run( + at.transcribe_audio( + MagicMock(), _agent(model="openai/gpt-4o"), "audio/ogg", "v.ogg", B64 + ) + ) + + assert text == "via openrouter" + kwargs = acompletion.await_args.kwargs + # model has a vendor segment -> prefixed verbatim (EVO-1684). + assert kwargs["model"] == "openrouter/openai/gpt-4o" + assert kwargs["api_base"] == "https://openrouter.ai/api/v1" + + +# ---- best-effort contract --------------------------------------------------- + + +def test_llm_failure_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + monkeypatch.setattr( + at.litellm, "acompletion", AsyncMock(side_effect=RuntimeError("boom")) + ) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_blank_transcript_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=" "))] + ) + monkeypatch.setattr(at.litellm, "acompletion", AsyncMock(return_value=resp)) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_missing_api_key_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(side_effect=ValueError("no key"))) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_undecodable_audio_returns_none(): + assert ( + _run( + at.transcribe_audio( + MagicMock(), _agent(), "audio/ogg", "v.ogg", "!!!not-base64!!!" + ) + ) + is None + ) + + +# ---- transcribe_audio_file (db / agent guards) ------------------------------ + + +def test_file_helper_none_db_returns_none(): + assert ( + _run(at.transcribe_audio_file(None, "agent-1", "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_file_helper_missing_agent_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_agent", AsyncMock(return_value=None)) + assert ( + _run( + at.transcribe_audio_file(MagicMock(), "agent-1", "audio/ogg", "v.ogg", B64) + ) + is None + ) + + +def test_file_helper_delegates_to_transcribe(monkeypatch): + agent = _agent() + monkeypatch.setattr(at, "get_agent", AsyncMock(return_value=agent)) + monkeypatch.setattr(at, "transcribe_audio", AsyncMock(return_value="delegated")) + text = _run( + at.transcribe_audio_file(MagicMock(), "agent-1", "audio/ogg", "v.ogg", B64) + ) + assert text == "delegated" diff --git a/tests/unit/test_media_file_parts.py b/tests/unit/test_media_file_parts.py index f45ef87..417e600 100644 --- a/tests/unit/test_media_file_parts.py +++ b/tests/unit/test_media_file_parts.py @@ -59,21 +59,46 @@ def test_image_is_appended_to_file_parts(): assert transcribed == [] -def test_audio_still_appended(): - parts, _ = _run( - _utils().process_files([_file("voice.ogg", "audio/ogg")], _artifacts(), "a", "e", "s") +def test_audio_is_not_inlined_as_a_model_part(): + # EVO-2227: audio never travels as a raw model part (google-adk emits an + # audio_url part litellm rejects -> 500). It is transcribed instead. Without + # a db (self.db unset here) transcription is a no-op, so parts AND transcript + # are both empty -- but the file is still archived. + artifacts = _artifacts() + parts, transcribed = _run( + _utils().process_files([_file("voice.ogg", "audio/ogg")], artifacts, "a", "e", "s") ) - assert len(parts) == 1 - assert parts[0].inline_data.mime_type == "audio/ogg" + assert parts == [] + assert transcribed == [] + artifacts.save_artifact.assert_awaited_once() # still kept for reference -def test_image_and_audio_both_appended(): +def test_only_the_image_is_inlined_next_to_audio(): + # The image inlines natively; the audio is routed to transcription (a no-op + # here) and never becomes a model part. parts, _ = _run( _utils().process_files( [_file("p.png", "image/png"), _file("v.ogg", "audio/ogg")], _artifacts(), "a", "e", "s" ) ) - assert len(parts) == 2 + assert [p.inline_data.mime_type for p in parts] == ["image/png"] + + +def test_audio_transcript_is_returned_and_not_inlined(monkeypatch): + # With a working transcriber, the audio yields transcribed text (which the + # runner folds into the message) and still no raw model part. + from unittest.mock import AsyncMock + + import src.services.adk.runners.runner_utils as ru + + monkeypatch.setattr(ru, "transcribe_audio_file", AsyncMock(return_value="olá do áudio")) + utils = _utils() + utils.db = MagicMock() # transcribe_audio_file is mocked; db is only passed through + parts, transcribed = _run( + utils.process_files([_file("voice.ogg", "audio/ogg")], _artifacts(), "agent", "ext", "sess") + ) + assert parts == [] + assert transcribed == ["olá do áudio"] def test_create_content_with_image_only_is_not_none(): @@ -128,13 +153,14 @@ def test_pdf_and_text_still_reach_the_model(): assert [p.inline_data.mime_type for p in parts] == ["application/pdf", "text/plain"] -def test_mime_parameters_do_not_break_the_check(): - parts, _ = _run( +def test_mime_parameters_do_not_break_audio_detection(): + # "audio/webm;codecs=opus" (what WhatsApp/browsers actually send) must still + # be recognized as audio -> routed to transcription, never inlined as a part. + parts, transcribed = _run( _utils().process_files([_file("v.webm", "audio/webm;codecs=opus")], _artifacts(), "a", "e", "s") ) - # The prefix still matches with the parameter attached, and ADK reads the - # verbatim value off the Blob, so the two agree. - assert [p.inline_data.mime_type for p in parts] == ["audio/webm;codecs=opus"] + assert parts == [] + assert transcribed == [] # no db here -> transcription is a no-op # The guard must judge the mime exactly as ADK will, because ADK is what raises. @@ -196,20 +222,23 @@ def test_inline_budget_is_per_request(monkeypatch): assert len(parts) == 1 -def test_every_forwarded_part_is_accepted_by_the_installed_adk(): - """Pins the allowlist to what google-adk actually converts. +def test_every_forwarded_part_is_accepted_by_adk_and_litellm(): + """Pins the allowlist to what BOTH layers accept. - `_get_content` is what raises the ValueError the runner turns into a 500. If - an ADK bump narrows the accepted set, this fails here instead of in front of - a customer. + `_get_content` (ADK) converts the parts; litellm's + `validate_chat_completion_user_messages` then guards the request. The old + test only exercised ADK, which happily produces an `audio_url` part -- and + litellm rejects that downstream with a 500. Audio is therefore no longer a + forwarded part (it is transcribed); the samples here are exactly what may + still travel as raw model parts, and both layers must accept them. """ + from litellm.utils import validate_chat_completion_user_messages + utils = _utils() samples = [ "image/png", "image/jpeg", "image/webp", - "audio/ogg", - "audio/mpeg", "video/mp4", "text/plain", "application/pdf", @@ -221,4 +250,22 @@ def test_every_forwarded_part_is_accepted_by_the_installed_adk(): ) ) assert len(parts) == len(samples) - _get_content(utils.create_content("", parts).parts) # must not raise + content = _get_content(utils.create_content("", parts).parts) # ADK: must not raise + # litellm layer: must not raise either (this is what audio_url tripped). + validate_chat_completion_user_messages([{"role": "user", "content": content}]) + + +def test_audio_inlined_as_a_part_would_be_rejected_by_litellm(): + """The reason audio is transcribed instead of inlined (EVO-2227). + + ADK converts an audio Blob into an `audio_url` content part, which is NOT in + litellm's ValidUserMessageContentTypes -> a 500 that costs the whole turn. + This pins that fact: if a future litellm starts accepting `audio_url`, this + fails and we can revisit native audio forwarding. + """ + from litellm.utils import validate_chat_completion_user_messages + + audio_part = Part(inline_data=Blob(mime_type="audio/ogg", data=b"bytes")) + content = _get_content([audio_part]) # ADK does not raise here... + with pytest.raises(Exception): # ...but litellm does. + validate_chat_completion_user_messages([{"role": "user", "content": content}]) From 3f1bfba9228017652faf4fd4a16b2e81bcba08c5 Mon Sep 17 00:00:00 2001 From: Matheus Pastorini Date: Mon, 27 Jul 2026 01:54:00 -0300 Subject: [PATCH 2/2] fix(agents): strip raw audio from poisoned session history (EVO-2227) New turns transcribe audio and never send it as a model part, but conversations that received audio BEFORE that fix persisted user events carrying the raw audio Blob. google-adk 1.19.0 turns those into an `audio_url` content part and litellm 1.68.2 rejects it (not in ValidUserMessageContentTypes) -> a 500 that breaks EVERY later turn in the conversation, audio OR text, until the session is cleared. Confirmed live: a conversation's ADK `events` table held 6 user events each ~9.7KB carrying the OggS audio inline_data; a fresh, correctly-transcribed turn still 500'd because ADK replays that history. Add a `before_model_callback` (strip_unsupported_audio_from_history) that drops any audio inline_data part from the outgoing LLM request, keeping the turn's text (e.g. the original caption). Defends already-poisoned histories and is defense-in-depth for any future path that lets audio through. - llm_agent_builder: new callback + wired via before_model_callback on every built agent. - tests/unit/test_strip_audio_history.py: 5 tests (audio dropped/text kept, audio-only -> [audio] placeholder, image/text untouched, mixed history, empty/None contents safe). Full tests/unit: 271 passed. --- src/services/adk/agents/llm_agent_builder.py | 38 ++++++++++ tests/unit/test_strip_audio_history.py | 74 ++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tests/unit/test_strip_audio_history.py diff --git a/src/services/adk/agents/llm_agent_builder.py b/src/services/adk/agents/llm_agent_builder.py index 13ed4d1..18367a1 100644 --- a/src/services/adk/agents/llm_agent_builder.py +++ b/src/services/adk/agents/llm_agent_builder.py @@ -316,6 +316,41 @@ async def update_current_time(callback_context: CallbackContext): callback_context.state["_datetime"] = now.isoformat() +async def strip_unsupported_audio_from_history(callback_context, llm_request): + """EVO-2227: drop raw audio parts from the outgoing LLM request. + + Audio is transcribed upstream (RunnerUtils.process_files) and never sent as a + model part on new turns. But conversations that received audio BEFORE that fix + persisted user events carrying the raw audio Blob; google-adk 1.19.0 turns + those into an ``audio_url`` content part and litellm 1.68.2 rejects it (not in + ValidUserMessageContentTypes) -> a 500 that breaks EVERY later turn in the + conversation, audio or text. This before_model_callback sanitizes the request + so a poisoned history can't kill the turn; the turn's text (e.g. the original + caption) is kept, only the audio bytes are dropped. + """ + contents = getattr(llm_request, "contents", None) or [] + for content in contents: + parts = getattr(content, "parts", None) + if not parts: + continue + kept = [] + dropped = False + for part in parts: + inline = getattr(part, "inline_data", None) + mime = (getattr(inline, "mime_type", "") or "") if inline else "" + if mime.lower().startswith("audio/"): + dropped = True + continue + kept.append(part) + if dropped: + if not kept: + from google.genai import types + + kept = [types.Part(text="[audio]")] + content.parts = kept + return None + + async def advanced_usage_tracker( callback_context: CallbackContext, llm_response: LlmResponse ) -> Optional[LlmResponse]: @@ -1138,6 +1173,9 @@ async def combined_callback(callback_context: CallbackContext): "description": agent.description, "tools": all_tools, "before_agent_callback": combined_callback, + # EVO-2227: sanitize any raw audio left in a pre-fix session history so + # it can't 500 the turn (google-adk audio_url part -> litellm reject). + "before_model_callback": strip_unsupported_audio_from_history, # "after_model_callback": advanced_usage_tracker, } diff --git a/tests/unit/test_strip_audio_history.py b/tests/unit/test_strip_audio_history.py new file mode 100644 index 0000000..c072fd8 --- /dev/null +++ b/tests/unit/test_strip_audio_history.py @@ -0,0 +1,74 @@ +"""EVO-2227: before_model_callback that strips raw audio from a poisoned history. + +Conversations that received audio before the transcription fix persisted user +events carrying the raw audio Blob. google-adk turns those into an `audio_url` +content part that litellm rejects -> a 500 on EVERY later turn. The callback +drops the audio bytes (keeping the turn's text) so the turn survives. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from google.genai import types + +from src.services.adk.agents.llm_agent_builder import ( + strip_unsupported_audio_from_history, +) + + +def _run(coro): + return asyncio.run(coro) + + +def _req(*contents): + return SimpleNamespace(contents=list(contents)) + + +def _audio_part(): + return types.Part(inline_data=types.Blob(mime_type="audio/opus", data=b"oggbytes")) + + +def _image_part(): + return types.Part(inline_data=types.Blob(mime_type="image/png", data=b"pngbytes")) + + +def test_audio_part_dropped_text_kept(): + content = types.Content( + role="user", + parts=[types.Part(text="Analyze the provided files"), _audio_part()], + ) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert [p.text for p in content.parts] == ["Analyze the provided files"] + assert all(p.inline_data is None for p in content.parts) + + +def test_audio_only_content_gets_text_placeholder(): + content = types.Content(role="user", parts=[_audio_part()]) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert len(content.parts) == 1 + assert content.parts[0].text == "[audio]" + assert content.parts[0].inline_data is None + + +def test_image_and_text_are_untouched(): + content = types.Content(role="user", parts=[types.Part(text="veja"), _image_part()]) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert content.parts[0].text == "veja" + assert content.parts[1].inline_data.mime_type == "image/png" + + +def test_mixed_history_only_audio_turns_are_cleaned(): + audio_turn = types.Content( + role="user", parts=[types.Part(text="oi"), _audio_part()] + ) + clean_turn = types.Content(role="model", parts=[types.Part(text="olá")]) + _run(strip_unsupported_audio_from_history(None, _req(audio_turn, clean_turn))) + assert all(p.inline_data is None for p in audio_turn.parts) + assert clean_turn.parts[0].text == "olá" + + +def test_empty_or_missing_contents_is_safe(): + _run(strip_unsupported_audio_from_history(None, _req())) + _run(strip_unsupported_audio_from_history(None, SimpleNamespace(contents=None)))