chore(llmobs): [MLOB-4813] log warning if litellm imported before being patched

ddtrace/llmobs/_llmobs.py

@@ -4,6 +4,7 @@
 import inspect
 import json
 import os
+import sys
 import time
 from typing import Any
 from typing import Callable
@@ -604,6 +605,8 @@ def enable(
             log.debug("%s already enabled", cls.__name__)
             return
 
+        cls._warn_if_litellm_was_imported()
+
         if os.getenv("DD_LLMOBS_ENABLED") and not asbool(os.getenv("DD_LLMOBS_ENABLED")):
             log.debug("LLMObs.enable() called when DD_LLMOBS_ENABLED is set to false or 0, not starting LLMObs service")
             return
@@ -704,6 +707,20 @@ def enable(
                 config._llmobs_ml_app,
             )
 
+    @staticmethod
+    def _warn_if_litellm_was_imported() -> None:
+        if "litellm" in sys.modules:
+            import litellm
+
+            if not getattr(litellm, "_datadog_patch", False):
+                log.warning(
+                    "LLMObs.enable() called after litellm was imported but before it was patched. "
+                    "This may cause tracing issues if you are importing patched methods like 'completion' directly. "
+                    "To ensure proper tracing, either run your application with ddtrace-run, "
+                    "call ddtrace.patch_all() before importing litellm, or "
+                    "enable LLMObs before importing other modules."
+                )
+
     def _on_asyncio_create_task(self, task_data: Dict[str, Any]) -> None:
         """Propagates llmobs active trace context across asyncio tasks."""
         task_data["llmobs_ctx"] = self._current_trace_context()

tests/contrib/litellm/conftest.py

@@ -1,4 +1,4 @@
-from litellm import Router
+import mock
 import pytest
 
 from ddtrace._trace.pin import Pin
@@ -81,4 +81,25 @@ def request_vcr_include_localhost():
 
 @pytest.fixture
 def router():
+    from litellm import Router
+
     yield Router(model_list=model_list)
+
+
+@pytest.fixture
+def mock_llmobs_logs():
+    with mock.patch("ddtrace.llmobs._llmobs.log") as m:
+        yield m
+        m.reset_mock()
+
+
+@pytest.fixture
+def clear_litellm_from_sys_modules():
+    import sys
+
+    litellm = sys.modules.get("litellm")
+    del sys.modules["litellm"]
+
+    yield
+
+    sys.modules["litellm"] = litellm

tests/contrib/litellm/test_litellm_llmobs.py

@@ -488,3 +488,41 @@ def test_completion_openai_enabled(
 
         assert len(llmobs_events) == 1
         assert llmobs_events[0]["name"] == "OpenAI.createChatCompletion" if not stream else "litellm.request"
+
+
+def test_enable_llmobs_after_litellm_was_imported(mock_llmobs_logs, clear_litellm_from_sys_modules):
+    """
+    Test that LLMObs.enable() logs a warning if litellm is imported before LLMObs.enable() is called.
+    """
+    from ddtrace.llmobs import LLMObs
+
+    LLMObs.disable()
+    import litellm  # noqa: F401
+
+    LLMObs.enable(ml_app="<ml-app-name>", integrations_enabled=False)
+    assert LLMObs.enabled
+    mock_llmobs_logs.warning.assert_called_once_with(
+        "LLMObs.enable() called after litellm was imported but before it was patched. "
+        "This may cause tracing issues if you are importing patched methods like 'completion' directly. "
+        "To ensure proper tracing, either run your application with ddtrace-run, "
+        "call ddtrace.patch_all() before importing litellm, or "
+        "enable LLMObs before importing other modules."
+    )
+
+    LLMObs.disable()
+
+
+def test_import_litellm_after_llmobs_was_enabled(mock_llmobs_logs, clear_litellm_from_sys_modules):
+    """
+    Test that LLMObs.enable() does not logs a warning if litellm is imported after LLMObs.enable() is called.
+    """
+    from ddtrace.llmobs import LLMObs
+
+    LLMObs.disable()
+    LLMObs.enable(ml_app="<ml-app-name>", integrations_enabled=False)
+    assert LLMObs.enabled
+    import litellm  # noqa: F401
+
+    mock_llmobs_logs.warning.assert_not_called()
+
+    LLMObs.disable()