feat(runners): multimodal audio transcription for the AI agent (EVO-2227 Fase 2)

src/services/adk/agents/llm_agent_builder.py

@@ -316,6 +316,41 @@ async def update_current_time(callback_context: CallbackContext):
     callback_context.state["_datetime"] = now.isoformat()
 
 
+async def strip_unsupported_audio_from_history(callback_context, llm_request):
+    """EVO-2227: drop raw audio parts from the outgoing LLM request.
+
+    Audio is transcribed upstream (RunnerUtils.process_files) and never sent as a
+    model part on new turns. But conversations that received audio BEFORE that fix
+    persisted user events carrying the raw audio Blob; google-adk 1.19.0 turns
+    those into an ``audio_url`` content part and litellm 1.68.2 rejects it (not in
+    ValidUserMessageContentTypes) -> a 500 that breaks EVERY later turn in the
+    conversation, audio or text. This before_model_callback sanitizes the request
+    so a poisoned history can't kill the turn; the turn's text (e.g. the original
+    caption) is kept, only the audio bytes are dropped.
+    """
+    contents = getattr(llm_request, "contents", None) or []
+    for content in contents:
+        parts = getattr(content, "parts", None)
+        if not parts:
+            continue
+        kept = []
+        dropped = False
+        for part in parts:
+            inline = getattr(part, "inline_data", None)
+            mime = (getattr(inline, "mime_type", "") or "") if inline else ""
+            if mime.lower().startswith("audio/"):
+                dropped = True
+                continue
+            kept.append(part)
+        if dropped:
+            if not kept:
+                from google.genai import types
+
+                kept = [types.Part(text="[audio]")]
+            content.parts = kept
+    return None
+
+
 async def advanced_usage_tracker(
     callback_context: CallbackContext, llm_response: LlmResponse
 ) -> Optional[LlmResponse]:
@@ -1138,6 +1173,9 @@ async def combined_callback(callback_context: CallbackContext):
             "description": agent.description,
             "tools": all_tools,
             "before_agent_callback": combined_callback,
+            # EVO-2227: sanitize any raw audio left in a pre-fix session history so
+            # it can't 500 the turn (google-adk audio_url part -> litellm reject).
+            "before_model_callback": strip_unsupported_audio_from_history,
             # "after_model_callback": advanced_usage_tracker,
         }
 

src/services/adk/runners/audio_transcription.py

@@ -0,0 +1,204 @@
+"""EVO-2227 (Fase 2): transcribe an incoming audio attachment with the agent's
+own multimodal model, so the transcript can be folded into the message text and
+understood by ANY answering LLM.
+
+Why not inline the raw audio to the agent instead? google-adk 1.19.0 emits an
+``audio_url`` content part for audio (``lite_llm.py`` ``_get_content``), and
+litellm 1.68.2 rejects ``audio_url`` -- it is not in
+``ValidUserMessageContentTypes`` (``text``/``image_url``/``input_audio``/
+``document``/``video_url``/``file``). The result is a 500 that costs the whole
+turn. So audio never travels as a raw model part; it is transcribed here and the
+text takes its place. Images still inline natively via ``image_url``.
+
+Provider routing (both accept WhatsApp opus/ogg without transcoding):
+- OpenAI family -> ``litellm.atranscription`` (whisper-1), a dedicated STT
+  endpoint that ingests opus/ogg/m4a/mp3/wav/webm directly.
+- Everything else that can hear audio in chat (Gemini) -> ``litellm.acompletion``
+  with an ``input_audio`` part; litellm's Gemini transform consumes it
+  (``vertex_ai/gemini/transformation.py``).
+
+Best-effort by contract: any failure (unsupported provider, network, quota)
+returns ``None`` so the turn survives on whatever text it already carries.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+from typing import Optional
+
+import litellm
+from sqlalchemy.orm import Session
+
+from src.models.models import Agent
+from src.services.adk.agents.agent_utils import get_api_key
+from src.services.agent_service import get_agent
+from src.utils.llm_model_routing import normalize_model_for_provider
+from src.utils.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# Kept short and directive so the model returns just the words, no preamble.
+_TRANSCRIBE_PROMPT = (
+    "Transcribe the following audio verbatim. Return only the transcription "
+    "text in the audio's own language, with no extra commentary."
+)
+
+# Model families whose keys route through OpenAI's STT endpoint. Matched against
+# the lowercased model identifier. Anything else is tried via chat input_audio.
+_OPENAI_MODEL_PREFIXES = ("gpt", "o1", "o3", "o4", "chatgpt", "whisper")
+
+# OpenAI's dedicated speech-to-text model. Reachable with the same key the agent
+# already uses; ingests opus/ogg directly (unlike chat input_audio, which is
+# wav/mp3 only).
+_OPENAI_TRANSCRIBE_MODEL = "whisper-1"
+
+# WhatsApp voice notes arrive labeled "audio/opus", but the bytes are an OGG
+# container (Opus codec) and Gemini's accepted audio MIME set lists "audio/ogg",
+# not "audio/opus". Map the codec label to the container so the chat provider
+# recognizes it. Only used on the chat (input_audio) path; whisper reads the file
+# regardless of the label.
+_CHAT_AUDIO_MIME_ALIASES = {
+    "audio/opus": "audio/ogg",
+    "audio/x-opus": "audio/ogg",
+}
+
+
+def _is_openai_family(model: str, provider: Optional[str]) -> bool:
+    """Whether to route transcription through OpenAI's STT endpoint.
+
+    OpenRouter keys never hit OpenAI's STT endpoint directly, so they fall to the
+    chat path regardless of the underlying vendor.
+    """
+    if provider == "openrouter":
+        return False
+    if provider == "openai":
+        return True
+    model_l = (model or "").lower()
+    if model_l.startswith("openai/"):
+        return True
+    return model_l.startswith(_OPENAI_MODEL_PREFIXES)
+
+
+async def transcribe_audio_file(
+    db: Optional[Session],
+    agent_id: str,
+    content_type: str,
+    filename: str,
+    data_b64: str,
+) -> Optional[str]:
+    """Resolve the agent's model/key and transcribe the base64 audio.
+
+    Returns the transcript text, or ``None`` when transcription is impossible or
+    fails (the caller keeps the turn either way).
+    """
+    if db is None or not agent_id:
+        return None
+    try:
+        agent = await get_agent(db, agent_id)
+    except Exception as e:  # get_agent can raise on a bad id / db hiccup
+        logger.warning(f"[AudioTranscription] could not load agent {agent_id}: {e}")
+        return None
+    if agent is None:
+        return None
+    return await transcribe_audio(db, agent, content_type, filename, data_b64)
+
+
+async def transcribe_audio(
+    db: Session, agent: Agent, content_type: str, filename: str, data_b64: str
+) -> Optional[str]:
+    """Transcribe one audio attachment with the agent's configured model."""
+    try:
+        raw_bytes = base64.b64decode(data_b64)
+    except Exception as e:
+        logger.warning(f"[AudioTranscription] undecodable audio {filename}: {e}")
+        return None
+    if not raw_bytes:
+        return None
+
+    try:
+        api_key, provider = await get_api_key(db, agent)
+    except Exception as e:
+        logger.warning(
+            f"[AudioTranscription] no usable API key for agent {agent.id}: {e}"
+        )
+        return None
+
+    model = agent.model or ""
+    try:
+        if _is_openai_family(model, provider):
+            text = await _transcribe_via_openai(
+                api_key, content_type, filename, raw_bytes
+            )
+        else:
+            text = await _transcribe_via_chat(
+                model, provider, api_key, content_type, raw_bytes
+            )
+    except Exception as e:
+        logger.warning(
+            f"[AudioTranscription] transcription failed for {filename}"
+            f" (model={model!r}, provider={provider!r}): {e}"
+        )
+        return None
+
+    text = (text or "").strip()
+    if not text:
+        logger.info(f"[AudioTranscription] empty transcript for {filename}")
+        return None
+    logger.info(
+        f"[AudioTranscription] transcribed {filename} ({len(raw_bytes)} bytes)"
+        f" -> {len(text)} chars"
+    )
+    return text
+
+
+async def _transcribe_via_openai(
+    api_key: str, content_type: str, filename: str, raw_bytes: bytes
+) -> Optional[str]:
+    """OpenAI STT (whisper-1). Ingests opus/ogg directly."""
+    audio = io.BytesIO(raw_bytes)
+    # The SDK derives the format from the file name's extension; keep the real
+    # one so an .ogg/.opus is not mistaken for something the endpoint rejects.
+    audio.name = filename or "audio.ogg"
+    resp = await litellm.atranscription(
+        model=_OPENAI_TRANSCRIBE_MODEL,
+        file=audio,
+        api_key=api_key,
+    )
+    # litellm returns a TranscriptionResponse with a .text attribute.
+    return getattr(resp, "text", None)
+
+
+async def _transcribe_via_chat(
+    model: str,
+    provider: Optional[str],
+    api_key: str,
+    content_type: str,
+    raw_bytes: bytes,
+) -> Optional[str]:
+    """Chat completion with an ``input_audio`` part (Gemini et al.)."""
+    norm_model, extra_kwargs = normalize_model_for_provider(model, provider)
+    # Normalize "audio/webm;codecs=opus" -> "audio/webm" for the data-uri header,
+    # then map codec labels (audio/opus) to the container MIME the provider knows.
+    mime = (content_type or "audio/ogg").split(";")[0].strip().lower() or "audio/ogg"
+    mime = _CHAT_AUDIO_MIME_ALIASES.get(mime, mime)
+    b64 = base64.b64encode(raw_bytes).decode("utf-8")
+    data_uri = f"data:{mime};base64,{b64}"
+    resp = await litellm.acompletion(
+        model=norm_model,
+        api_key=api_key,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": _TRANSCRIBE_PROMPT},
+                    {
+                        "type": "input_audio",
+                        "input_audio": {"data": data_uri, "format": mime},
+                    },
+                ],
+            }
+        ],
+        **extra_kwargs,
+    )
+    return resp.choices[0].message.content

src/services/adk/runners/runner_utils.py

@@ -39,6 +39,7 @@
 from src.core.exceptions import AgentNotFoundError
 from src.services.agent_service import get_agent
 from src.services.adk.agent_builder import AgentBuilder
+from src.services.adk.runners.audio_transcription import transcribe_audio_file
 from src.utils.adk_utils import extract_state_params
 from sqlalchemy.orm import Session
 from typing import Optional, List, Tuple, Dict, Any, Union
@@ -294,31 +295,60 @@ async def process_files(
                         )
                     )
 
-                    # EVO-2181: add the file to the content parts so the model
-                    # actually receives it. This append used to be gated behind
-                    # `if is_audio`, so an image was blobbed and saved to the
-                    # artifact store but never handed to the LLM -> the agent
-                    # replied "No content to process".
-                    #
-                    # It runs before save_artifact on purpose: the bytes are
-                    # already in hand, and a failure while storing them must not
-                    # cost the model its copy of the file and bring this very bug
-                    # back.
-                    skip_reason = self._inline_skip_reason(
-                        file_data.content_type, len(file_bytes), inlined_bytes
-                    )
-                    if skip_reason:
-                        logger.warning(
-                            f"File {file_data.filename} ({file_data.content_type}) not sent"
-                            f" to the model: {skip_reason}. Still saved as an artifact."
+                    if is_audio:
+                        # EVO-2227 (Fase 2): audio never travels as a raw model
+                        # part. google-adk 1.19.0 emits an `audio_url` content
+                        # part for audio and litellm 1.68.2 rejects it (not in
+                        # ValidUserMessageContentTypes) -> a 500 that costs the
+                        # whole turn. Instead transcribe it with the agent's own
+                        # multimodal model and let the text stand in, so ANY
+                        # answering LLM understands the voice note. Best-effort:
+                        # a None transcript leaves the turn on its remaining text.
+                        transcript = await transcribe_audio_file(
+                            getattr(self, "db", None),
+                            agent_id,
+                            file_data.content_type,
+                            file_data.filename,
+                            file_data.data,
                         )
+                        if transcript:
+                            transcribed_texts.append(transcript)
+                            logger.info(
+                                f"Transcribed audio {file_data.filename}"
+                                f" ({file_data.content_type}); text stands in for the file"
+                            )
+                        else:
+                            logger.warning(
+                                f"Audio {file_data.filename} could not be transcribed;"
+                                f" the turn continues on its remaining text"
+                            )
                     else:
-                        inlined_bytes += len(file_bytes)
-                        file_parts.append(file_part)
-                        logger.info(
-                            f"Added {'audio' if is_audio else 'file'} {file_data.filename}"
-                            f" ({file_data.content_type}) to content parts for LLM processing"
+                        # EVO-2181: add readable non-audio media (image/pdf/text/
+                        # video) to the content parts so the model actually
+                        # receives it. This append used to be gated behind
+                        # `if is_audio`, so an image was blobbed and saved to the
+                        # artifact store but never handed to the LLM -> the agent
+                        # replied "No content to process".
+                        #
+                        # It runs before save_artifact on purpose: the bytes are
+                        # already in hand, and a failure while storing them must
+                        # not cost the model its copy of the file and bring this
+                        # very bug back.
+                        skip_reason = self._inline_skip_reason(
+                            file_data.content_type, len(file_bytes), inlined_bytes
                         )
+                        if skip_reason:
+                            logger.warning(
+                                f"File {file_data.filename} ({file_data.content_type}) not sent"
+                                f" to the model: {skip_reason}. Still saved as an artifact."
+                            )
+                        else:
+                            inlined_bytes += len(file_bytes)
+                            file_parts.append(file_part)
+                            logger.info(
+                                f"Added file {file_data.filename}"
+                                f" ({file_data.content_type}) to content parts for LLM processing"
+                            )
 
                     # Always save to artifacts for reference
                     await artifacts_service.save_artifact(