Python: added WebRTC support for Azure OpenAI Realtime (#12078)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
Added support for WebRTC for Azure OpenAI Realtime models.

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

Author: eavanvalkenburg

python/samples/concepts/realtime/realtime_agent_with_function_calling_webrtc.py

@@ -8,9 +8,9 @@
 from samples.concepts.realtime.utils import AudioPlayerWebRTC, AudioRecorderWebRTC, check_audio_devices
 from semantic_kernel.connectors.ai import FunctionChoiceBehavior
 from semantic_kernel.connectors.ai.open_ai import (
+    AzureRealtimeWebRTC,
     ListenEvents,
     OpenAIRealtimeExecutionSettings,
-    OpenAIRealtimeWebRTC,
     TurnDetection,
 )
 from semantic_kernel.contents import ChatHistory, RealtimeTextEvent
@@ -26,11 +26,11 @@
 This simple sample demonstrates how to use the OpenAI Realtime API to create
 a agent that can listen and respond directly through audio.
 It requires installing:
-- semantic-kernel[realtime]
+- semantic-kernel
 - pyaudio
 - sounddevice
 - pydub
-e.g. pip install pyaudio sounddevice pydub semantic-kernel[realtime]
+e.g. pip install pyaudio sounddevice pydub semantic-kernel
 
 For more details of the exact setup, see the README.md in the realtime folder.
 """
@@ -79,7 +79,11 @@ async def main() -> None:
     # and can also be passed in the receive method
     # You can also pass in kernel, plugins, chat_history or settings here.
     # For WebRTC the audio_track is required
-    realtime_agent = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC(), plugins=[Helpers()])
+    realtime_agent = AzureRealtimeWebRTC(
+        audio_track=AudioRecorderWebRTC(),
+        region="swedencentral",
+        plugins=[Helpers()],
+    )
 
     # Create the settings for the session
     # The realtime api, does not use a system message, but takes instructions as a parameter for a session
@@ -109,12 +113,12 @@ async def main() -> None:
 
     # the context manager calls the create_session method on the client and starts listening to the audio stream
     async with (
-        audio_player,
         realtime_agent(
             settings=settings,
             chat_history=chat_history,
             create_response=True,
         ),
+        audio_player,
     ):
         async for event in realtime_agent.receive(audio_output_callback=audio_player.client_callback):
             match event:

python/samples/concepts/realtime/realtime_agent_with_function_calling_websocket.py

@@ -104,14 +104,14 @@ async def main() -> None:
 
     # the context manager calls the create_session method on the agent and starts listening to the audio stream
     async with (
-        audio_player,
         audio_recorder,
         realtime_agent(
             settings=settings,
             chat_history=chat_history,
             kernel=kernel,
             create_response=True,
         ),
+        audio_player,
     ):
         # the audio_output_callback can be added here or in the constructor
         # using this gives the smoothest experience

python/semantic_kernel/connectors/ai/open_ai/__init__.py

@@ -37,7 +37,7 @@
 from semantic_kernel.connectors.ai.open_ai.services._open_ai_realtime import ListenEvents, SendEvents
 from semantic_kernel.connectors.ai.open_ai.services.azure_audio_to_text import AzureAudioToText
 from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion
-from semantic_kernel.connectors.ai.open_ai.services.azure_realtime import AzureRealtimeWebsocket
+from semantic_kernel.connectors.ai.open_ai.services.azure_realtime import AzureRealtimeWebRTC, AzureRealtimeWebsocket
 from semantic_kernel.connectors.ai.open_ai.services.azure_text_completion import AzureTextCompletion
 from semantic_kernel.connectors.ai.open_ai.services.azure_text_embedding import AzureTextEmbedding
 from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_audio import AzureTextToAudio
@@ -68,6 +68,7 @@
     "AzureEmbeddingDependency",
     "AzureOpenAISettings",
     "AzureRealtimeExecutionSettings",
+    "AzureRealtimeWebRTC",
     "AzureRealtimeWebsocket",
     "AzureTextCompletion",
     "AzureTextEmbedding",

python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py

@@ -13,13 +13,16 @@ class InputAudioTranscription(KernelBaseModel):
     """Input audio transcription settings.
 
     Args:
-        model: The model to use for transcription, currently only "whisper-1" is supported.
+        model: The model to use for transcription, should be one of the following:
+            - whisper-1
+            - gpt-4o-transcribe
+            - gpt-4o-mini-transcribe
         language: The language of the audio, should be in ISO-639-1 format, like 'en'.
         prompt: An optional text to guide the model's style or continue a previous audio segment.
             The prompt should match the audio language.
     """
 
-    model: Literal["whisper-1"] | None = None
+    model: Literal["whisper-1", "gpt-4o-transcribe", "gpt-4o-mini-transcribe"] | None = None
     language: str | None = None
     prompt: str | None = None
 
@@ -28,19 +31,24 @@ class TurnDetection(KernelBaseModel):
     """Turn detection settings.
 
     Args:
-        type: The type of turn detection, currently only "server_vad" is supported.
-        threshold: The threshold for voice activity detection, should be between 0 and 1.
+        type: The type of turn detection, server_vad or semantic_vad.
+        create_response: Whether to create a response for each detected turn.
+        eagerness: The eagerness of the voice activity detection, can be low, medium, high, or auto,
+            used only for semantic_vad.
+        interrupt_response: Whether to interrupt the response for each detected turn.
         prefix_padding_ms: The padding before the detected voice activity, in milliseconds.
         silence_duration_ms: The duration of silence to detect the end of a turn, in milliseconds.
-        create_response: Whether to create a response for each detected turn.
+        threshold: The threshold for voice activity detection, should be between 0 and 1, only for server_vad.
 
     """
 
-    type: Literal["server_vad"] = "server_vad"
-    threshold: Annotated[float | None, Field(ge=0.0, le=1.0)] = None
+    type: Literal["server_vad", "semantic_vad"] = "server_vad"
+    create_response: bool | None = None
+    eagerness: Literal["low", "medium", "high", "auto"] | None = None
+    interrupt_response: bool | None = None
     prefix_padding_ms: Annotated[int | None, Field(ge=0)] = None
     silence_duration_ms: Annotated[int | None, Field(ge=0)] = None
-    create_response: bool | None = None
+    threshold: Annotated[float | None, Field(ge=0.0, le=1.0)] = None
 
 
 class OpenAIRealtimeExecutionSettings(PromptExecutionSettings):
@@ -68,8 +76,9 @@ class OpenAIRealtimeExecutionSettings(PromptExecutionSettings):
             "on the function choice configuration.",
         ),
     ] = None
-    temperature: Annotated[float | None, Field(ge=0.0, le=2.0)] = None
+    temperature: Annotated[float | None, Field(ge=0.6, le=1.2)] = None
     max_response_output_tokens: Annotated[int | Literal["inf"] | None, Field(gt=0)] = None
+    input_audio_noise_reduction: dict[Literal["type"], Literal["near_field", "far_field"]] | None = None
 
 
 class AzureRealtimeExecutionSettings(OpenAIRealtimeExecutionSettings):

python/semantic_kernel/connectors/ai/open_ai/services/_open_ai_realtime.py

@@ -10,11 +10,6 @@
 from enum import Enum
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
 
-if sys.version_info >= (3, 12):
-    from typing import override  # pragma: no cover
-else:
-    from typing_extensions import override  # pragma: no cover
-
 import numpy as np
 from aiohttp import ClientSession
 from aiortc import (
@@ -47,13 +42,15 @@
 from pydantic import Field, PrivateAttr
 
 from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
-from semantic_kernel.connectors.ai.function_calling_utils import (
-    prepare_settings_for_function_calling,
-)
+from semantic_kernel.connectors.ai.function_calling_utils import prepare_settings_for_function_calling
 from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType
+from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (
+    OpenAIRealtimeExecutionSettings,
+)
 from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler
 from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
 from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase
+from semantic_kernel.const import USER_AGENT
 from semantic_kernel.contents.audio_content import AudioContent
 from semantic_kernel.contents.chat_history import ChatHistory
 from semantic_kernel.contents.chat_message_content import ChatMessageContent
@@ -72,6 +69,7 @@
 from semantic_kernel.exceptions import ContentException
 from semantic_kernel.kernel import Kernel
 from semantic_kernel.utils.feature_stage_decorator import experimental
+from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT, prepend_semantic_kernel_to_user_agent
 
 if TYPE_CHECKING:
     from aiortc.mediastreams import MediaStreamTrack
@@ -84,7 +82,13 @@
     from semantic_kernel.contents.chat_history import ChatHistory
     from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
 
-logger: logging.Logger = logging.getLogger(__name__)
+
+if sys.version_info >= (3, 12):
+    from typing import override  # pragma: no cover
+else:
+    from typing_extensions import override  # pragma: no cover
+
+logger: logging.Logger = logging.getLogger("semantic_kernel.connectors.ai.open_ai.realtime")
 
 
 # region utils
@@ -649,10 +653,6 @@ async def send(self, event: RealtimeEvents, **kwargs: Any) -> None:
 
     @override
     def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
-        from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import (  # noqa
-            OpenAIRealtimeExecutionSettings,
-        )
-
         return OpenAIRealtimeExecutionSettings
 
     @override
@@ -725,14 +725,11 @@ async def create_session(
         try:
             ephemeral_token = await self._get_ephemeral_token()
             headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"}
+            headers = prepend_semantic_kernel_to_user_agent(headers)
 
             async with (
                 ClientSession() as session,
-                session.post(
-                    f"{self.client.beta.realtime._client.base_url}realtime?model={self.ai_model_id}",
-                    headers=headers,
-                    data=offer.sdp,
-                ) as response,
+                session.post(self._get_webrtc_url(), headers=headers, data=offer.sdp) as response,
             ):
                 if response.status not in [200, 201]:
                     error_text = await response.text()
@@ -813,15 +810,13 @@ async def _on_data(self, data: str) -> None:
 
     async def _get_ephemeral_token(self) -> str:
         """Get an ephemeral token from OpenAI."""
-        headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"}
-        data = {"model": self.ai_model_id, "voice": "echo"}
-
+        data = {"model": self.ai_model_id}
+        headers, url = self._get_ephemeral_token_headers_and_url()
+        headers = prepend_semantic_kernel_to_user_agent(headers)
         try:
             async with (
                 ClientSession() as session,
-                session.post(
-                    f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data
-                ) as response,
+                session.post(url, headers=headers, json=data) as response,
             ):
                 if response.status not in [200, 201]:
                     error_text = await response.text()
@@ -834,6 +829,17 @@ async def _get_ephemeral_token(self) -> str:
             logger.error(f"Failed to get ephemeral token: {e!s}")
             raise
 
+    def _get_ephemeral_token_headers_and_url(self) -> tuple[dict[str, str], str]:
+        """Get the headers for the ephemeral token."""
+        return {
+            "Authorization": f"Bearer {self.client.api_key}",
+            "Content-Type": "application/json",
+        }, f"{self.client.beta.realtime._client.base_url}/realtime/sessions"
+
+    def _get_webrtc_url(self) -> str:
+        """Get the WebRTC URL."""
+        return f"{self.client.beta.realtime._client.base_url}/realtime?model={self.ai_model_id}"
+
 
 # region Websocket
 
@@ -888,7 +894,9 @@ async def create_session(
         **kwargs: Any,
     ) -> None:
         """Create a session in the service."""
-        self.connection = await self.client.beta.realtime.connect(model=self.ai_model_id).enter()
+        self.connection = await self.client.beta.realtime.connect(
+            model=self.ai_model_id, extra_headers={USER_AGENT: SEMANTIC_KERNEL_USER_AGENT}
+        ).enter()
         self.connected.set()
         await self.update_session(settings=settings, chat_history=chat_history, **kwargs)
 

python/semantic_kernel/connectors/ai/open_ai/services/azure_config_base.py

@@ -105,6 +105,9 @@ def __init__(
             if deployment_name and ai_model_type != OpenAIModelTypes.REALTIME:
                 args["azure_deployment"] = deployment_name
 
+            if "websocket_base_url" in kwargs:
+                args["websocket_base_url"] = kwargs.pop("websocket_base_url")
+
             client = AsyncAzureOpenAI(**args)
         args = {
             "ai_model_id": deployment_name,