From e1e79143faea0d0d56e2f756e77d0fc07a68dd64 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Fri, 5 Dec 2025 13:59:12 -0500 Subject: [PATCH 1/7] log warning if litellm imported before being patched --- ddtrace/llmobs/_llmobs.py | 16 +++++++++ tests/contrib/litellm/conftest.py | 21 +++++++++++- tests/contrib/litellm/test_litellm_llmobs.py | 35 ++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 21d8174dc6d..a7e5306dc3c 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -4,6 +4,7 @@ import inspect import json import os +import sys import time from typing import Any from typing import Callable @@ -604,6 +605,8 @@ def enable( log.debug("%s already enabled", cls.__name__) return + cls._warn_if_litellm_was_imported() + if os.getenv("DD_LLMOBS_ENABLED") and not asbool(os.getenv("DD_LLMOBS_ENABLED")): log.debug("LLMObs.enable() called when DD_LLMOBS_ENABLED is set to false or 0, not starting LLMObs service") return @@ -704,6 +707,19 @@ def enable( config._llmobs_ml_app, ) + @staticmethod + def _warn_if_litellm_was_imported() -> None: + if "litellm" in sys.modules: + import litellm + if not getattr(litellm, "_datadog_patch", False): + log.warning( + "LLMObs.enable() called after litellm was imported but before it was patched. " + "This may cause tracing issues if you are importing patched methods like 'completion' directly. " + "To ensure proper tracing, either run your application with ddtrace-run, " + "call ddtrace.patch_all() before importing litellm, or " + "enable LLMObs before importing other modules." + ) + def _on_asyncio_create_task(self, task_data: Dict[str, Any]) -> None: """Propagates llmobs active trace context across asyncio tasks.""" task_data["llmobs_ctx"] = self._current_trace_context() diff --git a/tests/contrib/litellm/conftest.py b/tests/contrib/litellm/conftest.py index 62ea0c234fd..89c8436fd45 100644 --- a/tests/contrib/litellm/conftest.py +++ b/tests/contrib/litellm/conftest.py @@ -1,4 +1,4 @@ -from litellm import Router +import mock import pytest from ddtrace._trace.pin import Pin @@ -81,4 +81,23 @@ def request_vcr_include_localhost(): @pytest.fixture def router(): + from litellm import Router yield Router(model_list=model_list) + + +@pytest.fixture +def mock_llmobs_logs(): + with mock.patch("ddtrace.llmobs._llmobs.log") as m: + yield m + m.reset_mock() + +@pytest.fixture +def clear_litellm_from_sys_modules(): + import sys + + litellm = sys.modules.get("litellm") + del sys.modules["litellm"] + + yield + + sys.modules["litellm"] = litellm \ No newline at end of file diff --git a/tests/contrib/litellm/test_litellm_llmobs.py b/tests/contrib/litellm/test_litellm_llmobs.py index 5c039c56d55..b14bb589bd7 100644 --- a/tests/contrib/litellm/test_litellm_llmobs.py +++ b/tests/contrib/litellm/test_litellm_llmobs.py @@ -488,3 +488,38 @@ def test_completion_openai_enabled( assert len(llmobs_events) == 1 assert llmobs_events[0]["name"] == "OpenAI.createChatCompletion" if not stream else "litellm.request" + + +def test_enable_llmobs_after_litellm_was_imported(mock_llmobs_logs, clear_litellm_from_sys_modules): + """ + Test that LLMObs.enable() logs a warning if litellm is imported before LLMObs.enable() is called. + """ + from ddtrace.llmobs import LLMObs + LLMObs.disable() + import litellm + LLMObs.enable(ml_app="", integrations_enabled=False) + assert LLMObs.enabled + mock_llmobs_logs.warning.assert_called_once_with( + "LLMObs.enable() called after litellm was imported but before it was patched. " + "This may cause tracing issues if you are importing patched methods like 'completion' directly. " + "To ensure proper tracing, either run your application with ddtrace-run, " + "call ddtrace.patch_all() before importing litellm, or " + "enable LLMObs before importing other modules." + ) + + LLMObs.disable() + + +def test_import_litellm_after_llmobs_was_enabled(mock_llmobs_logs, clear_litellm_from_sys_modules): + """ + Test that LLMObs.enable() does not logs a warning if litellm is imported after LLMObs.enable() is called. + """ + from ddtrace.llmobs import LLMObs + LLMObs.disable() + LLMObs.enable(ml_app="", integrations_enabled=False) + assert LLMObs.enabled + import litellm + mock_llmobs_logs.warning.assert_not_called() + + LLMObs.disable() + From 582213ed76ae3f63bc4b3474ba35d9e4f939bfbc Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Fri, 5 Dec 2025 14:18:40 -0500 Subject: [PATCH 2/7] format --- ddtrace/llmobs/_llmobs.py | 1 + tests/contrib/litellm/conftest.py | 4 +++- tests/contrib/litellm/test_litellm_llmobs.py | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index a7e5306dc3c..59a2b3a5757 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -711,6 +711,7 @@ def enable( def _warn_if_litellm_was_imported() -> None: if "litellm" in sys.modules: import litellm + if not getattr(litellm, "_datadog_patch", False): log.warning( "LLMObs.enable() called after litellm was imported but before it was patched. " diff --git a/tests/contrib/litellm/conftest.py b/tests/contrib/litellm/conftest.py index 89c8436fd45..1ec39684e52 100644 --- a/tests/contrib/litellm/conftest.py +++ b/tests/contrib/litellm/conftest.py @@ -82,6 +82,7 @@ def request_vcr_include_localhost(): @pytest.fixture def router(): from litellm import Router + yield Router(model_list=model_list) @@ -91,6 +92,7 @@ def mock_llmobs_logs(): yield m m.reset_mock() + @pytest.fixture def clear_litellm_from_sys_modules(): import sys @@ -100,4 +102,4 @@ def clear_litellm_from_sys_modules(): yield - sys.modules["litellm"] = litellm \ No newline at end of file + sys.modules["litellm"] = litellm diff --git a/tests/contrib/litellm/test_litellm_llmobs.py b/tests/contrib/litellm/test_litellm_llmobs.py index b14bb589bd7..cebbc66f330 100644 --- a/tests/contrib/litellm/test_litellm_llmobs.py +++ b/tests/contrib/litellm/test_litellm_llmobs.py @@ -495,8 +495,10 @@ def test_enable_llmobs_after_litellm_was_imported(mock_llmobs_logs, clear_litell Test that LLMObs.enable() logs a warning if litellm is imported before LLMObs.enable() is called. """ from ddtrace.llmobs import LLMObs + LLMObs.disable() import litellm + LLMObs.enable(ml_app="", integrations_enabled=False) assert LLMObs.enabled mock_llmobs_logs.warning.assert_called_once_with( @@ -515,11 +517,12 @@ def test_import_litellm_after_llmobs_was_enabled(mock_llmobs_logs, clear_litellm Test that LLMObs.enable() does not logs a warning if litellm is imported after LLMObs.enable() is called. """ from ddtrace.llmobs import LLMObs + LLMObs.disable() LLMObs.enable(ml_app="", integrations_enabled=False) assert LLMObs.enabled import litellm + mock_llmobs_logs.warning.assert_not_called() LLMObs.disable() - From 43e087443179fa0f6d7db0d46af58f7d5d6fb1be Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Fri, 5 Dec 2025 14:23:38 -0500 Subject: [PATCH 3/7] suppress noqa errors --- tests/contrib/litellm/test_litellm_llmobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/contrib/litellm/test_litellm_llmobs.py b/tests/contrib/litellm/test_litellm_llmobs.py index cebbc66f330..8231fc9552d 100644 --- a/tests/contrib/litellm/test_litellm_llmobs.py +++ b/tests/contrib/litellm/test_litellm_llmobs.py @@ -497,7 +497,7 @@ def test_enable_llmobs_after_litellm_was_imported(mock_llmobs_logs, clear_litell from ddtrace.llmobs import LLMObs LLMObs.disable() - import litellm + import litellm # noqa: F401 LLMObs.enable(ml_app="", integrations_enabled=False) assert LLMObs.enabled @@ -521,7 +521,7 @@ def test_import_litellm_after_llmobs_was_enabled(mock_llmobs_logs, clear_litellm LLMObs.disable() LLMObs.enable(ml_app="", integrations_enabled=False) assert LLMObs.enabled - import litellm + import litellm # noqa: F401 mock_llmobs_logs.warning.assert_not_called() From f4a935c3c3d7a2b35ffa1b9674e86acb15f0e5d2 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Fri, 5 Dec 2025 17:42:34 -0500 Subject: [PATCH 4/7] switch to using subprocess tests --- tests/contrib/litellm/conftest.py | 20 -------- tests/contrib/litellm/test_litellm_llmobs.py | 52 ++++++++++---------- 2 files changed, 25 insertions(+), 47 deletions(-) diff --git a/tests/contrib/litellm/conftest.py b/tests/contrib/litellm/conftest.py index 1ec39684e52..be0754b82e7 100644 --- a/tests/contrib/litellm/conftest.py +++ b/tests/contrib/litellm/conftest.py @@ -1,4 +1,3 @@ -import mock import pytest from ddtrace._trace.pin import Pin @@ -84,22 +83,3 @@ def router(): from litellm import Router yield Router(model_list=model_list) - - -@pytest.fixture -def mock_llmobs_logs(): - with mock.patch("ddtrace.llmobs._llmobs.log") as m: - yield m - m.reset_mock() - - -@pytest.fixture -def clear_litellm_from_sys_modules(): - import sys - - litellm = sys.modules.get("litellm") - del sys.modules["litellm"] - - yield - - sys.modules["litellm"] = litellm diff --git a/tests/contrib/litellm/test_litellm_llmobs.py b/tests/contrib/litellm/test_litellm_llmobs.py index 8231fc9552d..3213019961e 100644 --- a/tests/contrib/litellm/test_litellm_llmobs.py +++ b/tests/contrib/litellm/test_litellm_llmobs.py @@ -489,40 +489,38 @@ def test_completion_openai_enabled( assert len(llmobs_events) == 1 assert llmobs_events[0]["name"] == "OpenAI.createChatCompletion" if not stream else "litellm.request" - -def test_enable_llmobs_after_litellm_was_imported(mock_llmobs_logs, clear_litellm_from_sys_modules): +def test_enable_llmobs_after_litellm_was_imported(run_python_code_in_subprocess): """ Test that LLMObs.enable() logs a warning if litellm is imported before LLMObs.enable() is called. """ - from ddtrace.llmobs import LLMObs - - LLMObs.disable() - import litellm # noqa: F401 - - LLMObs.enable(ml_app="", integrations_enabled=False) - assert LLMObs.enabled - mock_llmobs_logs.warning.assert_called_once_with( - "LLMObs.enable() called after litellm was imported but before it was patched. " - "This may cause tracing issues if you are importing patched methods like 'completion' directly. " - "To ensure proper tracing, either run your application with ddtrace-run, " - "call ddtrace.patch_all() before importing litellm, or " - "enable LLMObs before importing other modules." + _, err, _, _ = run_python_code_in_subprocess( + """ +import litellm +from ddtrace.llmobs import LLMObs +LLMObs.enable(ml_app="", integrations_enabled=False) +assert LLMObs.enabled +LLMObs.disable() +""" ) - LLMObs.disable() - + assert ( + "LLMObs.enable() called after litellm was imported but before it was patched" + ) in err.decode() -def test_import_litellm_after_llmobs_was_enabled(mock_llmobs_logs, clear_litellm_from_sys_modules): +def test_import_litellm_after_llmobs_was_enabled(run_python_code_in_subprocess): """ Test that LLMObs.enable() does not logs a warning if litellm is imported after LLMObs.enable() is called. """ - from ddtrace.llmobs import LLMObs - - LLMObs.disable() - LLMObs.enable(ml_app="", integrations_enabled=False) - assert LLMObs.enabled - import litellm # noqa: F401 - - mock_llmobs_logs.warning.assert_not_called() + _, err, _, _ = run_python_code_in_subprocess( + """ +from ddtrace.llmobs import LLMObs +LLMObs.enable(ml_app="", integrations_enabled=False) +assert LLMObs.enabled +import litellm +LLMObs.disable() +""" + ) - LLMObs.disable() + assert ( + "LLMObs.enable() called after litellm was imported but before it was patched" + ) not in err.decode() From d6d96a36b8ad9cca608f70012fa9e5cee4d37bf5 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Mon, 8 Dec 2025 10:54:15 -0500 Subject: [PATCH 5/7] format --- tests/contrib/litellm/test_litellm_llmobs.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/contrib/litellm/test_litellm_llmobs.py b/tests/contrib/litellm/test_litellm_llmobs.py index 3213019961e..619797dd983 100644 --- a/tests/contrib/litellm/test_litellm_llmobs.py +++ b/tests/contrib/litellm/test_litellm_llmobs.py @@ -489,6 +489,7 @@ def test_completion_openai_enabled( assert len(llmobs_events) == 1 assert llmobs_events[0]["name"] == "OpenAI.createChatCompletion" if not stream else "litellm.request" + def test_enable_llmobs_after_litellm_was_imported(run_python_code_in_subprocess): """ Test that LLMObs.enable() logs a warning if litellm is imported before LLMObs.enable() is called. @@ -503,9 +504,8 @@ def test_enable_llmobs_after_litellm_was_imported(run_python_code_in_subprocess) """ ) - assert ( - "LLMObs.enable() called after litellm was imported but before it was patched" - ) in err.decode() + assert ("LLMObs.enable() called after litellm was imported but before it was patched") in err.decode() + def test_import_litellm_after_llmobs_was_enabled(run_python_code_in_subprocess): """ @@ -521,6 +521,4 @@ def test_import_litellm_after_llmobs_was_enabled(run_python_code_in_subprocess): """ ) - assert ( - "LLMObs.enable() called after litellm was imported but before it was patched" - ) not in err.decode() + assert ("LLMObs.enable() called after litellm was imported but before it was patched") not in err.decode() From c1fea00168541ad01582df9d9494ac23a6d5272c Mon Sep 17 00:00:00 2001 From: ncybul <124532568+ncybul@users.noreply.github.com> Date: Wed, 10 Dec 2025 14:04:17 -0500 Subject: [PATCH 6/7] Update ddtrace/llmobs/_llmobs.py Co-authored-by: kyle --- ddtrace/llmobs/_llmobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 098bedbbca7..bb3ae8b1cfd 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -778,7 +778,7 @@ def _warn_if_litellm_was_imported() -> None: if not getattr(litellm, "_datadog_patch", False): log.warning( "LLMObs.enable() called after litellm was imported but before it was patched. " - "This may cause tracing issues if you are importing patched methods like 'completion' directly. " + "This may cause tracing issues if you are importing patched methods like 'litellm.completion' directly. " "To ensure proper tracing, either run your application with ddtrace-run, " "call ddtrace.patch_all() before importing litellm, or " "enable LLMObs before importing other modules." From d47b73afbb096b2814d1e633dc4ac4d9f78b0446 Mon Sep 17 00:00:00 2001 From: Nicole Cybul Date: Thu, 11 Dec 2025 11:47:22 -0500 Subject: [PATCH 7/7] format --- ddtrace/llmobs/_llmobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index bb3ae8b1cfd..e1de8a074d8 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -778,8 +778,8 @@ def _warn_if_litellm_was_imported() -> None: if not getattr(litellm, "_datadog_patch", False): log.warning( "LLMObs.enable() called after litellm was imported but before it was patched. " - "This may cause tracing issues if you are importing patched methods like 'litellm.completion' directly. " - "To ensure proper tracing, either run your application with ddtrace-run, " + "This may cause tracing issues if you are importing patched methods like 'litellm.completion' " + "directly. To ensure proper tracing, either run your application with ddtrace-run, " "call ddtrace.patch_all() before importing litellm, or " "enable LLMObs before importing other modules." )