Merge master into 2.0 branch (#2805)

* ref: Improve scrub_dict typing (#2768)

This change improves the typing of the scrub_dict method.

Previously, the scrub_dict method's type hints indicated that only dict[str, Any] was accepted as the parameter. However, the method is actually implemented to accept any object, since it checks the types of the parameters at runtime. Therefore, object is a more appropriate type hint for the parameter.

#2753 depends on this change for mypy to pass

* Propagate sentry-trace and baggage to huey tasks (#2792)

This PR enables passing `sentry-trace` and `baggage` headers to background tasks using the Huey task queue.

This allows easily correlating what happens inside a background task with whatever transaction (e.g. a user request in a Django application) queued the task in the first place.

Periodic tasks do not get these headers, because otherwise each execution of the periodic task would be tied to the same parent trace (the long-running worker process).

--- 

Co-authored-by: Anton Pirker <anton.pirker@sentry.io>

* OpenAI integration (#2791)

* OpenAI integration

* Fix linting errors

* Fix CI

* Fix lint

* Fix more CI issues

* Run tests on version pinned OpenAI too

* Fix pydantic issue in test

* Import type in TYPE_CHECKING gate

* PR feedback fixes

* Fix tiktoken test variant

* PII gate the request and response

* Rename set_data tags

* Move doc location

* Add "exclude prompts" flag as optional

* Change prompts to be excluded by default

* Set flag in tests

* Fix tiktoken tox.ini extra dash

* Change strip PII semantics

* More test coverage for PII

* notiktoken

---------

Co-authored-by: Anton Pirker <anton.pirker@sentry.io>

* Add a method for normalizing data passed to set_data (#2800)

* Discard open spans after 10 minutes (#2801)

OTel spans that are handled in the Sentry span processor can never be finished/closed. This leads to a memory leak. This change makes sure that open spans will be removed from memory after 10 minutes to prevent memory usage from growing constantly.

Fixes #2722

---------

Co-authored-by: Daniel Szoke <szokeasaurusrex@users.noreply.github.com>

* ref: Event Type (#2753)

Implements type hinting for Event via a TypedDict. This commit mainly adjusts type hints; however, there are also some minor code changes to make the code type-safe following the new changes.

Some items in the Event could have their types expanded by being defined as TypedDicts themselves. These items have been indicated with TODO comments.

Fixes GH-2357

* Fix mypy in `client.py`

* Fix functools import

* Fix CI config problem

... by running `python scripts/split-tox-gh-actions/split-tox-gh-actions.py`

---------

Co-authored-by: Christian Schneider <christian@cnschn.com>
Co-authored-by: Anton Pirker <anton.pirker@sentry.io>
Co-authored-by: colin-sentry <161344340+colin-sentry@users.noreply.github.com>

Author: 4 people

.github/workflows/test-integrations-data-processing.yml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6","3.7","3.8","3.11","3.12"]
+        python-version: ["3.6","3.7","3.8","3.9","3.11","3.12"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6
@@ -58,6 +58,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-huey-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+      - name: Test openai latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
       - name: Test rq latest
         run: |
           set -x # print commands that are executed
@@ -110,6 +114,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-huey" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+      - name: Test openai pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
       - name: Test rq pinned
         run: |
           set -x # print commands that are executed

mypy.ini

@@ -67,6 +67,8 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 [mypy-huey.*]
 ignore_missing_imports = True
+[mypy-openai.*]
+ignore_missing_imports = True
 [mypy-arq.*]
 ignore_missing_imports = True
 [mypy-grpc.*]

scripts/split-tox-gh-actions/split-tox-gh-actions.py

@@ -70,6 +70,7 @@
         "beam",
         "celery",
         "huey",
+        "openai",
         "rq",
     ],
     "Databases": [

sentry_sdk/_types.py

@@ -9,6 +9,10 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import MutableMapping
+
+    from datetime import datetime
+
     from types import TracebackType
     from typing import Any
     from typing import Callable
@@ -19,13 +23,69 @@
     from typing import Tuple
     from typing import Type
     from typing import Union
-    from typing_extensions import Literal
+    from typing_extensions import Literal, TypedDict
+
+    # "critical" is an alias of "fatal" recognized by Relay
+    LogLevelStr = Literal["fatal", "critical", "error", "warning", "info", "debug"]
+
+    Event = TypedDict(
+        "Event",
+        {
+            "breadcrumbs": dict[
+                Literal["values"], list[dict[str, Any]]
+            ],  # TODO: We can expand on this type
+            "check_in_id": str,
+            "contexts": dict[str, dict[str, object]],
+            "dist": str,
+            "duration": Optional[float],
+            "environment": str,
+            "errors": list[dict[str, Any]],  # TODO: We can expand on this type
+            "event_id": str,
+            "exception": dict[
+                Literal["values"], list[dict[str, Any]]
+            ],  # TODO: We can expand on this type
+            "extra": MutableMapping[str, object],
+            "fingerprint": list[str],
+            "level": LogLevelStr,
+            "logentry": Mapping[str, object],
+            "logger": str,
+            "measurements": dict[str, object],
+            "message": str,
+            "modules": dict[str, str],
+            "monitor_config": Mapping[str, object],
+            "monitor_slug": Optional[str],
+            "platform": Literal["python"],
+            "profile": object,  # Should be sentry_sdk.profiler.Profile, but we can't import that here due to circular imports
+            "release": str,
+            "request": dict[str, object],
+            "sdk": Mapping[str, object],
+            "server_name": str,
+            "spans": list[dict[str, object]],
+            "stacktrace": dict[
+                str, object
+            ],  # We access this key in the code, but I am unsure whether we ever set it
+            "start_timestamp": datetime,
+            "status": Optional[str],
+            "tags": MutableMapping[
+                str, str
+            ],  # Tags must be less than 200 characters each
+            "threads": dict[
+                Literal["values"], list[dict[str, Any]]
+            ],  # TODO: We can expand on this type
+            "timestamp": Optional[datetime],  # Must be set before sending the event
+            "transaction": str,
+            "transaction_info": Mapping[str, Any],  # TODO: We can expand on this type
+            "type": Literal["check_in", "transaction"],
+            "user": dict[str, object],
+            "_metrics_summary": dict[str, object],
+        },
+        total=False,
+    )
 
     ExcInfo = Tuple[
         Optional[Type[BaseException]], Optional[BaseException], Optional[TracebackType]
     ]
 
-    Event = Dict[str, Any]
     Hint = Dict[str, Any]
 
     Breadcrumb = Dict[str, Any]

sentry_sdk/api.py

@@ -27,6 +27,7 @@
         BreadcrumbHint,
         ExcInfo,
         MeasurementUnit,
+        LogLevelStr,
     )
     from sentry_sdk.scope import StartTransactionKwargs
     from sentry_sdk.tracing import Span
@@ -122,7 +123,7 @@ def capture_event(
 @scopemethod
 def capture_message(
     message,  # type: str
-    level=None,  # type: Optional[str]
+    level=None,  # type: Optional[LogLevelStr]
     scope=None,  # type: Optional[Any]
     **scope_kwargs,  # type: Any
 ):
@@ -257,7 +258,7 @@ def set_user(value):
 
 @scopemethod
 def set_level(value):
-    # type: (str) -> None
+    # type: (LogLevelStr) -> None
     return Scope.get_isolation_scope().set_level(value)
 
 

sentry_sdk/client.py

@@ -1,3 +1,8 @@
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping  # type: ignore[attr-defined]
+
 import os
 import uuid
 import random
@@ -32,7 +37,7 @@
 from sentry_sdk.utils import ContextVar
 from sentry_sdk.sessions import SessionFlusher
 from sentry_sdk.envelope import Envelope
-from sentry_sdk.profiler import has_profiling_enabled, setup_profiler
+from sentry_sdk.profiler import has_profiling_enabled, Profile, setup_profiler
 from sentry_sdk.scrubber import EventScrubber
 from sentry_sdk.monitor import Monitor
 from sentry_sdk.spotlight import setup_spotlight
@@ -460,7 +465,7 @@ def _prepare_event(
 
         for key in "release", "environment", "server_name", "dist":
             if event.get(key) is None and self.options[key] is not None:
-                event[key] = str(self.options[key]).strip()
+                event[key] = str(self.options[key]).strip()  # type: ignore[literal-required]
         if event.get("sdk") is None:
             sdk_info = dict(SDK_INFO)
             sdk_info["integrations"] = sorted(self.integrations.keys())
@@ -634,15 +639,16 @@ def _update_session_from_event(
             errored = True
             for error in exceptions:
                 mechanism = error.get("mechanism")
-                if mechanism and mechanism.get("handled") is False:
+                if isinstance(mechanism, Mapping) and mechanism.get("handled") is False:
                     crashed = True
                     break
 
         user = event.get("user")
 
         if session.user_agent is None:
             headers = (event.get("request") or {}).get("headers")
-            for k, v in (headers or {}).items():
+            headers_dict = headers if isinstance(headers, dict) else {}
+            for k, v in headers_dict.items():
                 if k.lower() == "user-agent":
                     user_agent = v
                     break
@@ -714,15 +720,15 @@ def capture_event(
         headers = {
             "event_id": event_opt["event_id"],
             "sent_at": format_timestamp(datetime.now(timezone.utc)),
-        }
+        }  # type: dict[str, object]
 
         if dynamic_sampling_context:
             headers["trace"] = dynamic_sampling_context
 
         envelope = Envelope(headers=headers)
 
         if is_transaction:
-            if profile is not None:
+            if isinstance(profile, Profile):
                 envelope.add_profile(profile.to_json(event_opt, self.options))
             envelope.add_transaction(event_opt)
         elif is_checkin:

sentry_sdk/consts.py

@@ -229,6 +229,8 @@ class OP:
     MIDDLEWARE_STARLITE = "middleware.starlite"
     MIDDLEWARE_STARLITE_RECEIVE = "middleware.starlite.receive"
     MIDDLEWARE_STARLITE_SEND = "middleware.starlite.send"
+    OPENAI_CHAT_COMPLETIONS_CREATE = "ai.chat_completions.create.openai"
+    OPENAI_EMBEDDINGS_CREATE = "ai.embeddings.create.openai"
     QUEUE_SUBMIT_ARQ = "queue.submit.arq"
     QUEUE_TASK_ARQ = "queue.task.arq"
     QUEUE_SUBMIT_CELERY = "queue.submit.celery"

sentry_sdk/crons/api.py

@@ -6,6 +6,7 @@
 
 if TYPE_CHECKING:
     from typing import Any, Dict, Optional
+    from sentry_sdk._types import Event
 
 
 def _create_check_in_event(
@@ -15,7 +16,7 @@ def _create_check_in_event(
     duration_s=None,
     monitor_config=None,
 ):
-    # type: (Optional[str], Optional[str], Optional[str], Optional[float], Optional[Dict[str, Any]]) -> Dict[str, Any]
+    # type: (Optional[str], Optional[str], Optional[str], Optional[float], Optional[Dict[str, Any]]) -> Event
     options = Hub.current.client.options if Hub.current.client else {}
     check_in_id = check_in_id or uuid.uuid4().hex  # type: str
 
@@ -27,7 +28,7 @@ def _create_check_in_event(
         "duration": duration_s,
         "environment": options.get("environment", None),
         "release": options.get("release", None),
-    }
+    }  # type: Event
 
     if monitor_config:
         check_in["monitor_config"] = monitor_config

sentry_sdk/hub.py

@@ -41,6 +41,7 @@
         Breadcrumb,
         BreadcrumbHint,
         ExcInfo,
+        LogLevelStr,
     )
     from sentry_sdk.consts import ClientConstructor
     from sentry_sdk.scope import StartTransactionKwargs
@@ -347,7 +348,7 @@ def capture_event(self, event, hint=None, scope=None, **scope_kwargs):
         return last_event_id
 
     def capture_message(self, message, level=None, scope=None, **scope_kwargs):
-        # type: (str, Optional[str], Optional[Scope], Any) -> Optional[str]
+        # type: (str, Optional[LogLevelStr], Optional[Scope], Any) -> Optional[str]
         """
         .. deprecated:: 2.0.0
             This function is deprecated and will be removed in a future release.

sentry_sdk/integrations/__init__.py

@@ -86,6 +86,7 @@ def iter_default_integrations(with_auto_enabling_integrations):
     "sentry_sdk.integrations.httpx.HttpxIntegration",
     "sentry_sdk.integrations.huey.HueyIntegration",
     "sentry_sdk.integrations.loguru.LoguruIntegration",
+    "sentry_sdk.integrations.openai.OpenAIIntegration",
     "sentry_sdk.integrations.pymongo.PyMongoIntegration",
     "sentry_sdk.integrations.pyramid.PyramidIntegration",
     "sentry_sdk.integrations.quart.QuartIntegration",