diff --git a/src/services/adk/agents/llm_agent_builder.py b/src/services/adk/agents/llm_agent_builder.py index 13ed4d1..18367a1 100644 --- a/src/services/adk/agents/llm_agent_builder.py +++ b/src/services/adk/agents/llm_agent_builder.py @@ -316,6 +316,41 @@ async def update_current_time(callback_context: CallbackContext): callback_context.state["_datetime"] = now.isoformat() +async def strip_unsupported_audio_from_history(callback_context, llm_request): + """EVO-2227: drop raw audio parts from the outgoing LLM request. + + Audio is transcribed upstream (RunnerUtils.process_files) and never sent as a + model part on new turns. But conversations that received audio BEFORE that fix + persisted user events carrying the raw audio Blob; google-adk 1.19.0 turns + those into an ``audio_url`` content part and litellm 1.68.2 rejects it (not in + ValidUserMessageContentTypes) -> a 500 that breaks EVERY later turn in the + conversation, audio or text. This before_model_callback sanitizes the request + so a poisoned history can't kill the turn; the turn's text (e.g. the original + caption) is kept, only the audio bytes are dropped. + """ + contents = getattr(llm_request, "contents", None) or [] + for content in contents: + parts = getattr(content, "parts", None) + if not parts: + continue + kept = [] + dropped = False + for part in parts: + inline = getattr(part, "inline_data", None) + mime = (getattr(inline, "mime_type", "") or "") if inline else "" + if mime.lower().startswith("audio/"): + dropped = True + continue + kept.append(part) + if dropped: + if not kept: + from google.genai import types + + kept = [types.Part(text="[audio]")] + content.parts = kept + return None + + async def advanced_usage_tracker( callback_context: CallbackContext, llm_response: LlmResponse ) -> Optional[LlmResponse]: @@ -1138,6 +1173,9 @@ async def combined_callback(callback_context: CallbackContext): "description": agent.description, "tools": all_tools, "before_agent_callback": combined_callback, + # EVO-2227: sanitize any raw audio left in a pre-fix session history so + # it can't 500 the turn (google-adk audio_url part -> litellm reject). + "before_model_callback": strip_unsupported_audio_from_history, # "after_model_callback": advanced_usage_tracker, } diff --git a/src/services/adk/runners/audio_transcription.py b/src/services/adk/runners/audio_transcription.py new file mode 100644 index 0000000..6ac6308 --- /dev/null +++ b/src/services/adk/runners/audio_transcription.py @@ -0,0 +1,204 @@ +"""EVO-2227 (Fase 2): transcribe an incoming audio attachment with the agent's +own multimodal model, so the transcript can be folded into the message text and +understood by ANY answering LLM. + +Why not inline the raw audio to the agent instead? google-adk 1.19.0 emits an +``audio_url`` content part for audio (``lite_llm.py`` ``_get_content``), and +litellm 1.68.2 rejects ``audio_url`` -- it is not in +``ValidUserMessageContentTypes`` (``text``/``image_url``/``input_audio``/ +``document``/``video_url``/``file``). The result is a 500 that costs the whole +turn. So audio never travels as a raw model part; it is transcribed here and the +text takes its place. Images still inline natively via ``image_url``. + +Provider routing (both accept WhatsApp opus/ogg without transcoding): +- OpenAI family -> ``litellm.atranscription`` (whisper-1), a dedicated STT + endpoint that ingests opus/ogg/m4a/mp3/wav/webm directly. +- Everything else that can hear audio in chat (Gemini) -> ``litellm.acompletion`` + with an ``input_audio`` part; litellm's Gemini transform consumes it + (``vertex_ai/gemini/transformation.py``). + +Best-effort by contract: any failure (unsupported provider, network, quota) +returns ``None`` so the turn survives on whatever text it already carries. +""" + +from __future__ import annotations + +import base64 +import io +from typing import Optional + +import litellm +from sqlalchemy.orm import Session + +from src.models.models import Agent +from src.services.adk.agents.agent_utils import get_api_key +from src.services.agent_service import get_agent +from src.utils.llm_model_routing import normalize_model_for_provider +from src.utils.logger import setup_logger + +logger = setup_logger(__name__) + +# Kept short and directive so the model returns just the words, no preamble. +_TRANSCRIBE_PROMPT = ( + "Transcribe the following audio verbatim. Return only the transcription " + "text in the audio's own language, with no extra commentary." +) + +# Model families whose keys route through OpenAI's STT endpoint. Matched against +# the lowercased model identifier. Anything else is tried via chat input_audio. +_OPENAI_MODEL_PREFIXES = ("gpt", "o1", "o3", "o4", "chatgpt", "whisper") + +# OpenAI's dedicated speech-to-text model. Reachable with the same key the agent +# already uses; ingests opus/ogg directly (unlike chat input_audio, which is +# wav/mp3 only). +_OPENAI_TRANSCRIBE_MODEL = "whisper-1" + +# WhatsApp voice notes arrive labeled "audio/opus", but the bytes are an OGG +# container (Opus codec) and Gemini's accepted audio MIME set lists "audio/ogg", +# not "audio/opus". Map the codec label to the container so the chat provider +# recognizes it. Only used on the chat (input_audio) path; whisper reads the file +# regardless of the label. +_CHAT_AUDIO_MIME_ALIASES = { + "audio/opus": "audio/ogg", + "audio/x-opus": "audio/ogg", +} + + +def _is_openai_family(model: str, provider: Optional[str]) -> bool: + """Whether to route transcription through OpenAI's STT endpoint. + + OpenRouter keys never hit OpenAI's STT endpoint directly, so they fall to the + chat path regardless of the underlying vendor. + """ + if provider == "openrouter": + return False + if provider == "openai": + return True + model_l = (model or "").lower() + if model_l.startswith("openai/"): + return True + return model_l.startswith(_OPENAI_MODEL_PREFIXES) + + +async def transcribe_audio_file( + db: Optional[Session], + agent_id: str, + content_type: str, + filename: str, + data_b64: str, +) -> Optional[str]: + """Resolve the agent's model/key and transcribe the base64 audio. + + Returns the transcript text, or ``None`` when transcription is impossible or + fails (the caller keeps the turn either way). + """ + if db is None or not agent_id: + return None + try: + agent = await get_agent(db, agent_id) + except Exception as e: # get_agent can raise on a bad id / db hiccup + logger.warning(f"[AudioTranscription] could not load agent {agent_id}: {e}") + return None + if agent is None: + return None + return await transcribe_audio(db, agent, content_type, filename, data_b64) + + +async def transcribe_audio( + db: Session, agent: Agent, content_type: str, filename: str, data_b64: str +) -> Optional[str]: + """Transcribe one audio attachment with the agent's configured model.""" + try: + raw_bytes = base64.b64decode(data_b64) + except Exception as e: + logger.warning(f"[AudioTranscription] undecodable audio {filename}: {e}") + return None + if not raw_bytes: + return None + + try: + api_key, provider = await get_api_key(db, agent) + except Exception as e: + logger.warning( + f"[AudioTranscription] no usable API key for agent {agent.id}: {e}" + ) + return None + + model = agent.model or "" + try: + if _is_openai_family(model, provider): + text = await _transcribe_via_openai( + api_key, content_type, filename, raw_bytes + ) + else: + text = await _transcribe_via_chat( + model, provider, api_key, content_type, raw_bytes + ) + except Exception as e: + logger.warning( + f"[AudioTranscription] transcription failed for {filename}" + f" (model={model!r}, provider={provider!r}): {e}" + ) + return None + + text = (text or "").strip() + if not text: + logger.info(f"[AudioTranscription] empty transcript for {filename}") + return None + logger.info( + f"[AudioTranscription] transcribed {filename} ({len(raw_bytes)} bytes)" + f" -> {len(text)} chars" + ) + return text + + +async def _transcribe_via_openai( + api_key: str, content_type: str, filename: str, raw_bytes: bytes +) -> Optional[str]: + """OpenAI STT (whisper-1). Ingests opus/ogg directly.""" + audio = io.BytesIO(raw_bytes) + # The SDK derives the format from the file name's extension; keep the real + # one so an .ogg/.opus is not mistaken for something the endpoint rejects. + audio.name = filename or "audio.ogg" + resp = await litellm.atranscription( + model=_OPENAI_TRANSCRIBE_MODEL, + file=audio, + api_key=api_key, + ) + # litellm returns a TranscriptionResponse with a .text attribute. + return getattr(resp, "text", None) + + +async def _transcribe_via_chat( + model: str, + provider: Optional[str], + api_key: str, + content_type: str, + raw_bytes: bytes, +) -> Optional[str]: + """Chat completion with an ``input_audio`` part (Gemini et al.).""" + norm_model, extra_kwargs = normalize_model_for_provider(model, provider) + # Normalize "audio/webm;codecs=opus" -> "audio/webm" for the data-uri header, + # then map codec labels (audio/opus) to the container MIME the provider knows. + mime = (content_type or "audio/ogg").split(";")[0].strip().lower() or "audio/ogg" + mime = _CHAT_AUDIO_MIME_ALIASES.get(mime, mime) + b64 = base64.b64encode(raw_bytes).decode("utf-8") + data_uri = f"data:{mime};base64,{b64}" + resp = await litellm.acompletion( + model=norm_model, + api_key=api_key, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": _TRANSCRIBE_PROMPT}, + { + "type": "input_audio", + "input_audio": {"data": data_uri, "format": mime}, + }, + ], + } + ], + **extra_kwargs, + ) + return resp.choices[0].message.content diff --git a/src/services/adk/runners/runner_utils.py b/src/services/adk/runners/runner_utils.py index f354f8c..1110015 100644 --- a/src/services/adk/runners/runner_utils.py +++ b/src/services/adk/runners/runner_utils.py @@ -39,6 +39,7 @@ from src.core.exceptions import AgentNotFoundError from src.services.agent_service import get_agent from src.services.adk.agent_builder import AgentBuilder +from src.services.adk.runners.audio_transcription import transcribe_audio_file from src.utils.adk_utils import extract_state_params from sqlalchemy.orm import Session from typing import Optional, List, Tuple, Dict, Any, Union @@ -294,31 +295,60 @@ async def process_files( ) ) - # EVO-2181: add the file to the content parts so the model - # actually receives it. This append used to be gated behind - # `if is_audio`, so an image was blobbed and saved to the - # artifact store but never handed to the LLM -> the agent - # replied "No content to process". - # - # It runs before save_artifact on purpose: the bytes are - # already in hand, and a failure while storing them must not - # cost the model its copy of the file and bring this very bug - # back. - skip_reason = self._inline_skip_reason( - file_data.content_type, len(file_bytes), inlined_bytes - ) - if skip_reason: - logger.warning( - f"File {file_data.filename} ({file_data.content_type}) not sent" - f" to the model: {skip_reason}. Still saved as an artifact." + if is_audio: + # EVO-2227 (Fase 2): audio never travels as a raw model + # part. google-adk 1.19.0 emits an `audio_url` content + # part for audio and litellm 1.68.2 rejects it (not in + # ValidUserMessageContentTypes) -> a 500 that costs the + # whole turn. Instead transcribe it with the agent's own + # multimodal model and let the text stand in, so ANY + # answering LLM understands the voice note. Best-effort: + # a None transcript leaves the turn on its remaining text. + transcript = await transcribe_audio_file( + getattr(self, "db", None), + agent_id, + file_data.content_type, + file_data.filename, + file_data.data, ) + if transcript: + transcribed_texts.append(transcript) + logger.info( + f"Transcribed audio {file_data.filename}" + f" ({file_data.content_type}); text stands in for the file" + ) + else: + logger.warning( + f"Audio {file_data.filename} could not be transcribed;" + f" the turn continues on its remaining text" + ) else: - inlined_bytes += len(file_bytes) - file_parts.append(file_part) - logger.info( - f"Added {'audio' if is_audio else 'file'} {file_data.filename}" - f" ({file_data.content_type}) to content parts for LLM processing" + # EVO-2181: add readable non-audio media (image/pdf/text/ + # video) to the content parts so the model actually + # receives it. This append used to be gated behind + # `if is_audio`, so an image was blobbed and saved to the + # artifact store but never handed to the LLM -> the agent + # replied "No content to process". + # + # It runs before save_artifact on purpose: the bytes are + # already in hand, and a failure while storing them must + # not cost the model its copy of the file and bring this + # very bug back. + skip_reason = self._inline_skip_reason( + file_data.content_type, len(file_bytes), inlined_bytes ) + if skip_reason: + logger.warning( + f"File {file_data.filename} ({file_data.content_type}) not sent" + f" to the model: {skip_reason}. Still saved as an artifact." + ) + else: + inlined_bytes += len(file_bytes) + file_parts.append(file_part) + logger.info( + f"Added file {file_data.filename}" + f" ({file_data.content_type}) to content parts for LLM processing" + ) # Always save to artifacts for reference await artifacts_service.save_artifact( diff --git a/tests/unit/test_audio_transcription.py b/tests/unit/test_audio_transcription.py new file mode 100644 index 0000000..f448f10 --- /dev/null +++ b/tests/unit/test_audio_transcription.py @@ -0,0 +1,220 @@ +"""EVO-2227 (Fase 2): transcribe audio with the agent's own multimodal model. + +Covers the provider routing (OpenAI STT vs chat input_audio), the best-effort +contract (any failure -> None so the turn survives), and the input shaping. +""" + +from __future__ import annotations + +import asyncio +import base64 +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import src.services.adk.runners.audio_transcription as at + +B64 = base64.b64encode(b"opus-bytes").decode() + + +def _run(coro): + return asyncio.run(coro) + + +def _agent(model="gemini/gemini-2.0-flash", agent_id="agent-1"): + return SimpleNamespace(id=agent_id, model=model, api_key_id=None, config={}) + + +# ---- _is_openai_family ------------------------------------------------------ + + +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("gpt-4o", None, True), + ("gpt-4.1-mini", None, True), + ("o3-mini", None, True), + ("openai/gpt-4o", None, True), + ("anything", "openai", True), + ("gemini/gemini-2.0-flash", None, False), + ("gemini-1.5-pro", None, False), + ("gpt-4o", "openrouter", False), # openrouter never hits OpenAI STT + ("claude-3-5-sonnet", None, False), + ], +) +def test_is_openai_family(model, provider, expected): + assert at._is_openai_family(model, provider) is expected + + +# ---- routing ---------------------------------------------------------------- + + +def test_openai_family_routes_to_whisper(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("sk-x", "openai"))) + atranscription = AsyncMock(return_value=SimpleNamespace(text="olá do whisper")) + monkeypatch.setattr(at.litellm, "atranscription", atranscription) + monkeypatch.setattr( + at.litellm, + "acompletion", + AsyncMock(side_effect=AssertionError("should not be called")), + ) + + text = _run( + at.transcribe_audio( + MagicMock(), _agent(model="gpt-4o"), "audio/ogg", "v.ogg", B64 + ) + ) + + assert text == "olá do whisper" + assert atranscription.await_count == 1 + kwargs = atranscription.await_args.kwargs + assert kwargs["model"] == "whisper-1" + assert kwargs["api_key"] == "sk-x" + + +def test_gemini_routes_to_chat_input_audio(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="olá do gemini"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + monkeypatch.setattr( + at.litellm, + "atranscription", + AsyncMock(side_effect=AssertionError("should not be called")), + ) + + text = _run( + at.transcribe_audio( + MagicMock(), + _agent(model="gemini/gemini-2.0-flash"), + "audio/webm;codecs=opus", + "v.webm", + B64, + ) + ) + + assert text == "olá do gemini" + msg = acompletion.await_args.kwargs["messages"][0] + part = msg["content"][1] + assert part["type"] == "input_audio" + # mime parameter stripped for the data-uri header + assert part["input_audio"]["data"].startswith("data:audio/webm;base64,") + assert part["input_audio"]["format"] == "audio/webm" + + +def test_opus_label_is_mapped_to_ogg_container_for_chat(monkeypatch): + # WhatsApp sends "audio/opus" but the bytes are an OGG container; Gemini + # knows "audio/ogg", not "audio/opus". + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/opus", "v.ogg", B64)) + + part = acompletion.await_args.kwargs["messages"][0]["content"][1] + assert part["input_audio"]["data"].startswith("data:audio/ogg;base64,") + assert part["input_audio"]["format"] == "audio/ogg" + + +def test_openrouter_uses_chat_path_with_api_base(monkeypatch): + monkeypatch.setattr( + at, "get_api_key", AsyncMock(return_value=("or-key", "openrouter")) + ) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="via openrouter"))] + ) + acompletion = AsyncMock(return_value=resp) + monkeypatch.setattr(at.litellm, "acompletion", acompletion) + + text = _run( + at.transcribe_audio( + MagicMock(), _agent(model="openai/gpt-4o"), "audio/ogg", "v.ogg", B64 + ) + ) + + assert text == "via openrouter" + kwargs = acompletion.await_args.kwargs + # model has a vendor segment -> prefixed verbatim (EVO-1684). + assert kwargs["model"] == "openrouter/openai/gpt-4o" + assert kwargs["api_base"] == "https://openrouter.ai/api/v1" + + +# ---- best-effort contract --------------------------------------------------- + + +def test_llm_failure_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + monkeypatch.setattr( + at.litellm, "acompletion", AsyncMock(side_effect=RuntimeError("boom")) + ) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_blank_transcript_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(return_value=("gm-key", None))) + resp = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=" "))] + ) + monkeypatch.setattr(at.litellm, "acompletion", AsyncMock(return_value=resp)) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_missing_api_key_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_api_key", AsyncMock(side_effect=ValueError("no key"))) + assert ( + _run(at.transcribe_audio(MagicMock(), _agent(), "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_undecodable_audio_returns_none(): + assert ( + _run( + at.transcribe_audio( + MagicMock(), _agent(), "audio/ogg", "v.ogg", "!!!not-base64!!!" + ) + ) + is None + ) + + +# ---- transcribe_audio_file (db / agent guards) ------------------------------ + + +def test_file_helper_none_db_returns_none(): + assert ( + _run(at.transcribe_audio_file(None, "agent-1", "audio/ogg", "v.ogg", B64)) + is None + ) + + +def test_file_helper_missing_agent_returns_none(monkeypatch): + monkeypatch.setattr(at, "get_agent", AsyncMock(return_value=None)) + assert ( + _run( + at.transcribe_audio_file(MagicMock(), "agent-1", "audio/ogg", "v.ogg", B64) + ) + is None + ) + + +def test_file_helper_delegates_to_transcribe(monkeypatch): + agent = _agent() + monkeypatch.setattr(at, "get_agent", AsyncMock(return_value=agent)) + monkeypatch.setattr(at, "transcribe_audio", AsyncMock(return_value="delegated")) + text = _run( + at.transcribe_audio_file(MagicMock(), "agent-1", "audio/ogg", "v.ogg", B64) + ) + assert text == "delegated" diff --git a/tests/unit/test_media_file_parts.py b/tests/unit/test_media_file_parts.py index f45ef87..417e600 100644 --- a/tests/unit/test_media_file_parts.py +++ b/tests/unit/test_media_file_parts.py @@ -59,21 +59,46 @@ def test_image_is_appended_to_file_parts(): assert transcribed == [] -def test_audio_still_appended(): - parts, _ = _run( - _utils().process_files([_file("voice.ogg", "audio/ogg")], _artifacts(), "a", "e", "s") +def test_audio_is_not_inlined_as_a_model_part(): + # EVO-2227: audio never travels as a raw model part (google-adk emits an + # audio_url part litellm rejects -> 500). It is transcribed instead. Without + # a db (self.db unset here) transcription is a no-op, so parts AND transcript + # are both empty -- but the file is still archived. + artifacts = _artifacts() + parts, transcribed = _run( + _utils().process_files([_file("voice.ogg", "audio/ogg")], artifacts, "a", "e", "s") ) - assert len(parts) == 1 - assert parts[0].inline_data.mime_type == "audio/ogg" + assert parts == [] + assert transcribed == [] + artifacts.save_artifact.assert_awaited_once() # still kept for reference -def test_image_and_audio_both_appended(): +def test_only_the_image_is_inlined_next_to_audio(): + # The image inlines natively; the audio is routed to transcription (a no-op + # here) and never becomes a model part. parts, _ = _run( _utils().process_files( [_file("p.png", "image/png"), _file("v.ogg", "audio/ogg")], _artifacts(), "a", "e", "s" ) ) - assert len(parts) == 2 + assert [p.inline_data.mime_type for p in parts] == ["image/png"] + + +def test_audio_transcript_is_returned_and_not_inlined(monkeypatch): + # With a working transcriber, the audio yields transcribed text (which the + # runner folds into the message) and still no raw model part. + from unittest.mock import AsyncMock + + import src.services.adk.runners.runner_utils as ru + + monkeypatch.setattr(ru, "transcribe_audio_file", AsyncMock(return_value="olá do áudio")) + utils = _utils() + utils.db = MagicMock() # transcribe_audio_file is mocked; db is only passed through + parts, transcribed = _run( + utils.process_files([_file("voice.ogg", "audio/ogg")], _artifacts(), "agent", "ext", "sess") + ) + assert parts == [] + assert transcribed == ["olá do áudio"] def test_create_content_with_image_only_is_not_none(): @@ -128,13 +153,14 @@ def test_pdf_and_text_still_reach_the_model(): assert [p.inline_data.mime_type for p in parts] == ["application/pdf", "text/plain"] -def test_mime_parameters_do_not_break_the_check(): - parts, _ = _run( +def test_mime_parameters_do_not_break_audio_detection(): + # "audio/webm;codecs=opus" (what WhatsApp/browsers actually send) must still + # be recognized as audio -> routed to transcription, never inlined as a part. + parts, transcribed = _run( _utils().process_files([_file("v.webm", "audio/webm;codecs=opus")], _artifacts(), "a", "e", "s") ) - # The prefix still matches with the parameter attached, and ADK reads the - # verbatim value off the Blob, so the two agree. - assert [p.inline_data.mime_type for p in parts] == ["audio/webm;codecs=opus"] + assert parts == [] + assert transcribed == [] # no db here -> transcription is a no-op # The guard must judge the mime exactly as ADK will, because ADK is what raises. @@ -196,20 +222,23 @@ def test_inline_budget_is_per_request(monkeypatch): assert len(parts) == 1 -def test_every_forwarded_part_is_accepted_by_the_installed_adk(): - """Pins the allowlist to what google-adk actually converts. +def test_every_forwarded_part_is_accepted_by_adk_and_litellm(): + """Pins the allowlist to what BOTH layers accept. - `_get_content` is what raises the ValueError the runner turns into a 500. If - an ADK bump narrows the accepted set, this fails here instead of in front of - a customer. + `_get_content` (ADK) converts the parts; litellm's + `validate_chat_completion_user_messages` then guards the request. The old + test only exercised ADK, which happily produces an `audio_url` part -- and + litellm rejects that downstream with a 500. Audio is therefore no longer a + forwarded part (it is transcribed); the samples here are exactly what may + still travel as raw model parts, and both layers must accept them. """ + from litellm.utils import validate_chat_completion_user_messages + utils = _utils() samples = [ "image/png", "image/jpeg", "image/webp", - "audio/ogg", - "audio/mpeg", "video/mp4", "text/plain", "application/pdf", @@ -221,4 +250,22 @@ def test_every_forwarded_part_is_accepted_by_the_installed_adk(): ) ) assert len(parts) == len(samples) - _get_content(utils.create_content("", parts).parts) # must not raise + content = _get_content(utils.create_content("", parts).parts) # ADK: must not raise + # litellm layer: must not raise either (this is what audio_url tripped). + validate_chat_completion_user_messages([{"role": "user", "content": content}]) + + +def test_audio_inlined_as_a_part_would_be_rejected_by_litellm(): + """The reason audio is transcribed instead of inlined (EVO-2227). + + ADK converts an audio Blob into an `audio_url` content part, which is NOT in + litellm's ValidUserMessageContentTypes -> a 500 that costs the whole turn. + This pins that fact: if a future litellm starts accepting `audio_url`, this + fails and we can revisit native audio forwarding. + """ + from litellm.utils import validate_chat_completion_user_messages + + audio_part = Part(inline_data=Blob(mime_type="audio/ogg", data=b"bytes")) + content = _get_content([audio_part]) # ADK does not raise here... + with pytest.raises(Exception): # ...but litellm does. + validate_chat_completion_user_messages([{"role": "user", "content": content}]) diff --git a/tests/unit/test_strip_audio_history.py b/tests/unit/test_strip_audio_history.py new file mode 100644 index 0000000..c072fd8 --- /dev/null +++ b/tests/unit/test_strip_audio_history.py @@ -0,0 +1,74 @@ +"""EVO-2227: before_model_callback that strips raw audio from a poisoned history. + +Conversations that received audio before the transcription fix persisted user +events carrying the raw audio Blob. google-adk turns those into an `audio_url` +content part that litellm rejects -> a 500 on EVERY later turn. The callback +drops the audio bytes (keeping the turn's text) so the turn survives. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from google.genai import types + +from src.services.adk.agents.llm_agent_builder import ( + strip_unsupported_audio_from_history, +) + + +def _run(coro): + return asyncio.run(coro) + + +def _req(*contents): + return SimpleNamespace(contents=list(contents)) + + +def _audio_part(): + return types.Part(inline_data=types.Blob(mime_type="audio/opus", data=b"oggbytes")) + + +def _image_part(): + return types.Part(inline_data=types.Blob(mime_type="image/png", data=b"pngbytes")) + + +def test_audio_part_dropped_text_kept(): + content = types.Content( + role="user", + parts=[types.Part(text="Analyze the provided files"), _audio_part()], + ) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert [p.text for p in content.parts] == ["Analyze the provided files"] + assert all(p.inline_data is None for p in content.parts) + + +def test_audio_only_content_gets_text_placeholder(): + content = types.Content(role="user", parts=[_audio_part()]) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert len(content.parts) == 1 + assert content.parts[0].text == "[audio]" + assert content.parts[0].inline_data is None + + +def test_image_and_text_are_untouched(): + content = types.Content(role="user", parts=[types.Part(text="veja"), _image_part()]) + _run(strip_unsupported_audio_from_history(None, _req(content))) + assert content.parts[0].text == "veja" + assert content.parts[1].inline_data.mime_type == "image/png" + + +def test_mixed_history_only_audio_turns_are_cleaned(): + audio_turn = types.Content( + role="user", parts=[types.Part(text="oi"), _audio_part()] + ) + clean_turn = types.Content(role="model", parts=[types.Part(text="olá")]) + _run(strip_unsupported_audio_from_history(None, _req(audio_turn, clean_turn))) + assert all(p.inline_data is None for p in audio_turn.parts) + assert clean_turn.parts[0].text == "olá" + + +def test_empty_or_missing_contents_is_safe(): + _run(strip_unsupported_audio_from_history(None, _req())) + _run(strip_unsupported_audio_from_history(None, SimpleNamespace(contents=None)))