From 6e7f19624bb23b35ef5263f3c1f4eb7bf5673521 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Tue, 3 Feb 2026 12:58:33 -0800 Subject: [PATCH 1/3] Add OpenTelemetry tracing examples for OpenAI Agents workflows This commit adds a comprehensive set of OTEL tracing examples demonstrating three progressive patterns: 1. Basic: Pure automatic instrumentation - plugin handles everything 2. Custom Spans: trace() + custom_span() for logical grouping 3. Direct API: trace() + custom_span() + direct OTEL tracer for detailed instrumentation Key additions: - openai_agents/otel_tracing/ directory with README and three workflow examples - Worker and client scripts for each pattern - Documentation covering setup, configuration, and troubleshooting - Added required OTEL dependencies to pyproject.toml Pattern clarification: - Never use trace() in client code - Use trace() in workflows when using custom_span() (patterns 2 & 3) - Plugin automatically creates root trace for pure automatic instrumentation Co-Authored-By: Claude Sonnet 4.5 --- openai_agents/otel_tracing/README.md | 237 ++++++++++++++++++ openai_agents/otel_tracing/__init__.py | 0 openai_agents/otel_tracing/run_otel_basic.py | 62 +++++ .../otel_tracing/run_otel_custom_spans.py | 59 +++++ .../otel_tracing/run_otel_direct_api.py | 73 ++++++ openai_agents/otel_tracing/run_worker.py | 82 ++++++ .../otel_tracing/workflows/__init__.py | 0 .../workflows/otel_basic_workflow.py | 58 +++++ .../workflows/otel_custom_spans_workflow.py | 78 ++++++ .../workflows/otel_direct_api_workflow.py | 128 ++++++++++ pyproject.toml | 4 +- 11 files changed, 780 insertions(+), 1 deletion(-) create mode 100644 openai_agents/otel_tracing/README.md create mode 100644 openai_agents/otel_tracing/__init__.py create mode 100755 openai_agents/otel_tracing/run_otel_basic.py create mode 100755 openai_agents/otel_tracing/run_otel_custom_spans.py create mode 100755 openai_agents/otel_tracing/run_otel_direct_api.py create mode 100755 openai_agents/otel_tracing/run_worker.py create mode 100644 openai_agents/otel_tracing/workflows/__init__.py create mode 100644 openai_agents/otel_tracing/workflows/otel_basic_workflow.py create mode 100644 openai_agents/otel_tracing/workflows/otel_custom_spans_workflow.py create mode 100644 openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py diff --git a/openai_agents/otel_tracing/README.md b/openai_agents/otel_tracing/README.md new file mode 100644 index 00000000..4630f2cd --- /dev/null +++ b/openai_agents/otel_tracing/README.md @@ -0,0 +1,237 @@ +# OpenTelemetry (OTEL) Tracing for OpenAI Agents + +This example demonstrates how to instrument OpenAI Agents workflows with OpenTelemetry (OTEL) for distributed tracing and observability. + +Traces can be exported to any OTEL-compatible backend such as Jaeger, Grafana Tempo, Datadog, New Relic, or other tracing systems. + +## Overview + +The Temporal OpenAI Agents SDK provides built-in OTEL integration that: +- **Automatically instruments** agent runs, model calls, and activities +- **Is replay-safe** - spans are only exported when workflows actually complete (not during replay) +- **Provides deterministic IDs** - consistent span/trace IDs across workflow replays +- **Supports multiple exporters** - send traces to multiple backends simultaneously + +## Two Instrumentation Patterns + +### 1. Automatic Instrumentation (Recommended for Most Users) + +The SDK automatically creates spans for: +- Agent execution +- Model invocations (as Temporal activities) +- Tool/activity calls +- Workflow lifecycle events + +**Use this when:** You want visibility into agent behavior without custom instrumentation. + +See `run_otel_basic.py` for an example. + +### 2. Direct OTEL API Usage (Advanced) + +You can use the OpenTelemetry API directly in workflows to: +- Instrument custom business logic +- Add domain-specific spans and attributes +- Integrate with organization-wide OTEL conventions +- Monitor performance of specific operations + +**Use this when:** You need to trace custom workflow logic beyond agent/model calls. + +See `run_otel_direct_api.py` for an example. + +## Quick Start + +### Prerequisites + +Ensure you have an OTEL collector or backend running. For local testing with Grafana Tempo: + +```bash +git clone https://github.com/grafana/tempo.git +cd tempo/example/docker-compose/local +mkdir tempo-data/ +docker compose up -d +``` + +View traces at: http://localhost:3000/explore + +### Install Dependencies + +```bash +uv sync +``` + +### Start the Worker + +```bash +uv run openai_agents/otel_tracing/run_worker.py +``` + +### Run Examples + +In separate terminals: + +**Basic automatic instrumentation:** +```bash +uv run openai_agents/otel_tracing/run_otel_basic.py +``` + +**Direct OTEL API usage:** +```bash +uv run openai_agents/otel_tracing/run_otel_direct_api.py +``` + +## Implementation Details + +### Plugin Configuration + +Configure OTEL exporters in the `OpenAIAgentsPlugin`: + +```python +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.contrib.openai_agents import OpenAIAgentsPlugin + +client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=30) + ), + otel_exporters=[ + OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + ], + add_temporal_spans=False, # Optional: exclude Temporal internal spans + ), + ], +) +``` + +### Key Parameters + +**`otel_exporters`**: List of OTEL span exporters +- `OTLPSpanExporter` - OTLP protocol (most common) +- `InMemorySpanExporter` - For testing +- `ConsoleSpanExporter` - Debug output +- Multiple exporters supported simultaneously + +**`add_temporal_spans`**: Whether to include Temporal internal spans (default: `True`) +- `False` - Cleaner traces focused on agent logic +- `True` - Full visibility including Temporal internals (startWorkflow, executeActivity, etc.) + +### Direct OTEL API Usage + +**CRITICAL REQUIREMENTS** for using `opentelemetry.trace` API directly in workflows: + +#### 1. Wrap in `custom_span()` from Agents SDK + +Direct OTEL calls **MUST** be wrapped in `custom_span()` to establish the bridge between Agent SDK trace context and OTEL context: + +```python +from agents import custom_span +import opentelemetry.trace + +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self): + # ✅ CORRECT - custom_span establishes OTEL context + with custom_span("My workflow logic"): + tracer = opentelemetry.trace.get_tracer(__name__) + with tracer.start_as_current_span("custom-instrumentation") as span: + span.set_attribute("business.metric", 42) + # This span will be properly parented + result = await my_business_logic() + + # ❌ WRONG - becomes orphaned root span, disconnected from trace + tracer = opentelemetry.trace.get_tracer(__name__) + with tracer.start_as_current_span("orphaned-span"): + # No connection to the agent trace! + pass +``` + +#### 2. Configure Sandbox Passthrough + +The `opentelemetry` module must be allowed in the workflow sandbox: + +```python +from temporalio.worker import Worker +from temporalio.worker.workflow_sandbox import ( + SandboxedWorkflowRunner, + SandboxRestrictions, +) + +worker = Worker( + client, + task_queue="otel-task-queue", + workflows=[MyWorkflow], + workflow_runner=SandboxedWorkflowRunner( + SandboxRestrictions.default.with_passthrough_modules("opentelemetry") + ), +) +``` + +### Trace Context Propagation + +Traces automatically propagate through the system: + +``` +Client trace + └─> Workflow execution + ├─> Agent span + │ └─> Model activity + ├─> Custom span (if using direct API) + └─> Tool activity +``` + +- **Client → Workflow**: Trace context propagates when starting workflow within `trace()` block +- **Workflow → Activity**: Context automatically propagates to activities +- **Replay-safe**: Spans only export on actual completion, not during replay + +### Environment Configuration + +Set the OTEL service name (optional): +```bash +export OTEL_SERVICE_NAME=my-agent-service +``` + +## Common Use Cases + +### Basic Monitoring +Use automatic instrumentation to: +- Monitor agent performance +- Debug agent behavior +- Track model API usage +- Identify bottlenecks + +### Custom Business Logic +Use direct OTEL API to: +- Instrument domain-specific operations +- Add business metrics as span attributes +- Create logical groupings of related operations +- Integrate with existing observability stack + +### Production Observability +- Export to multiple backends (e.g., Jaeger for dev, Datadog for prod) +- Use `add_temporal_spans=False` for cleaner production traces +- Add custom attributes for filtering/grouping in your observability tool + +## Troubleshooting + +### Spans not appearing in backend +- Verify OTLP endpoint is accessible +- Check that backend is configured to accept OTLP +- Ensure workflow completes (spans only export on completion) + +### Direct OTEL spans become root spans +- Verify you're wrapping calls in `custom_span()` +- Check sandbox passthrough is configured +- Ensure you're within an existing trace context + +### Duplicate spans across replays +- This is expected behavior during development with workflow cache +- Spans are only exported once per actual execution, not per replay + +## Related Examples + +- [grafana-tempo-openai-example](../../../../grafana-tempo-openai-example/) - End-to-end observability demo +- [basic/](../basic/) - Simple agent examples without OTEL +- [financial_research_agent](../financial_research_agent/) - Complex multi-agent example diff --git a/openai_agents/otel_tracing/__init__.py b/openai_agents/otel_tracing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openai_agents/otel_tracing/run_otel_basic.py b/openai_agents/otel_tracing/run_otel_basic.py new file mode 100755 index 00000000..c45452f0 --- /dev/null +++ b/openai_agents/otel_tracing/run_otel_basic.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Client for basic OTEL tracing example. + +This demonstrates the simplest OTEL integration - automatic instrumentation +of agent/model/activity spans without any custom code. + +The worker configuration handles all OTEL setup. This client just executes +the workflow normally. +""" + +import asyncio +import uuid +from datetime import timedelta + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.client import Client +from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin + +from openai_agents.otel_tracing.workflows.otel_basic_workflow import OtelBasicWorkflow + + +async def main(): + # Configure OTLP exporter (same as worker) + exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + + client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=60) + ), + otel_exporters=[exporter], + add_temporal_spans=False, + ), + ], + ) + + question = "What's the weather like in Tokyo?" + print(f"Question: {question}\n") + + result = await client.execute_workflow( + OtelBasicWorkflow.run, + question, + id=f"otel-basic-workflow-{uuid.uuid4()}", + task_queue="otel-task-queue", + ) + + print(f"Answer: {result}\n") + print("✓ Workflow completed") + print("\nView traces at:") + print(" - Grafana Tempo: http://localhost:3000/explore") + print(" - Jaeger: http://localhost:16686/") + print("\nExpected spans in trace:") + print(" - Workflow execution") + print(" - Agent run (Weather Assistant)") + print(" - Model invocation (activity)") + print(" - Tool call (get_weather activity)") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/openai_agents/otel_tracing/run_otel_custom_spans.py b/openai_agents/otel_tracing/run_otel_custom_spans.py new file mode 100755 index 00000000..95d079a4 --- /dev/null +++ b/openai_agents/otel_tracing/run_otel_custom_spans.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Client for custom spans OTEL example. + +This demonstrates using custom_span() to create logical groupings in traces +while still benefiting from automatic instrumentation. +""" + +import asyncio +import uuid +from datetime import timedelta + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.client import Client +from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin + +from openai_agents.otel_tracing.workflows.otel_custom_spans_workflow import ( + OtelCustomSpansWorkflow, +) + + +async def main(): + # Configure OTLP exporter (same as worker) + exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + + client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=60) + ), + otel_exporters=[exporter], + add_temporal_spans=False, + ), + ], + ) + + print("Checking weather for multiple cities...\n") + + result = await client.execute_workflow( + OtelCustomSpansWorkflow.run, + id=f"otel-custom-spans-workflow-{uuid.uuid4()}", + task_queue="otel-task-queue", + ) + + print(f"Results:\n{result}\n") + print("✓ Workflow completed") + print("\nView traces at:") + print(" - Grafana Tempo: http://localhost:3000/explore") + print(" - Jaeger: http://localhost:16686/") + print("\nExpected spans in trace:") + print(" - Multi-city weather check (custom_span grouping)") + print(" - Agent runs for Tokyo, Paris, New York (3 agents)") + print(" - Model invocations (activities)") + print(" - Tool calls (get_weather activities)") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/openai_agents/otel_tracing/run_otel_direct_api.py b/openai_agents/otel_tracing/run_otel_direct_api.py new file mode 100755 index 00000000..1e3a6873 --- /dev/null +++ b/openai_agents/otel_tracing/run_otel_direct_api.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Client for direct OTEL API usage example. + +This demonstrates using the OpenTelemetry API directly in workflows to +instrument custom business logic, add domain-specific spans, and set +custom attributes. + +The workflow uses custom_span() to establish OTEL context, then creates +custom spans for validation, business logic, and formatting operations. +""" + +import asyncio +import uuid +from datetime import timedelta + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.client import Client +from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin + +from openai_agents.otel_tracing.workflows.otel_direct_api_workflow import ( + OtelDirectApiWorkflow, +) + + +async def main(): + # Configure OTLP exporter (same as worker) + exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + + client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=60) + ), + otel_exporters=[exporter], + add_temporal_spans=False, + ), + ], + ) + + city = "Paris" + print(f"Getting travel recommendation for: {city}\n") + + result = await client.execute_workflow( + OtelDirectApiWorkflow.run, + city, + id=f"otel-direct-api-workflow-{uuid.uuid4()}", + task_queue="otel-task-queue", + ) + + print(f"Result:\n{result}\n") + print("✓ Workflow completed") + print("\nView traces at:") + print(" - Grafana Tempo: http://localhost:3000/explore") + print(" - Jaeger: http://localhost:16686/") + print("\nExpected spans in trace:") + print(" - Travel recommendation workflow (custom_span)") + print(" - validate-input (direct OTEL span)") + print(" - Agent run (Travel Weather Assistant)") + print(" - fetch-weather-info (direct OTEL span)") + print(" - Model invocation (activity)") + print(" - Tool call (get_weather activity)") + print(" - calculate-travel-score (direct OTEL span)") + print(" - format-response (direct OTEL span)") + print("\nLook for custom attributes on spans:") + print(" - input.city, validation.result") + print(" - request.city, response.length") + print(" - travel.score, travel.recommendation") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/openai_agents/otel_tracing/run_worker.py b/openai_agents/otel_tracing/run_worker.py new file mode 100755 index 00000000..f2410618 --- /dev/null +++ b/openai_agents/otel_tracing/run_worker.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Worker for OTEL tracing examples. + +This worker demonstrates OTEL configuration for both automatic instrumentation +and direct OTEL API usage patterns. + +Configuration: +- OTLP exporter for sending traces to OTEL backend +- Sandbox passthrough for opentelemetry module (required for direct API usage) +- Optional add_temporal_spans parameter for controlling span verbosity +""" + +import asyncio +from datetime import timedelta + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.client import Client +from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin +from temporalio.worker import Worker +from temporalio.worker.workflow_sandbox import ( + SandboxedWorkflowRunner, + SandboxRestrictions, +) + +from openai_agents.otel_tracing.workflows.otel_basic_workflow import ( + OtelBasicWorkflow, + get_weather, +) +from openai_agents.otel_tracing.workflows.otel_direct_api_workflow import ( + OtelDirectApiWorkflow, +) + + +async def main(): + # Configure OTLP exporter + # Default endpoint is localhost:4317 for Grafana Tempo/Jaeger + # Adjust endpoint for your OTEL backend (e.g., Datadog, New Relic, etc.) + exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + + client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=60) + ), + otel_exporters=[exporter], + # Optional: Set to False to exclude Temporal internal spans for cleaner traces + add_temporal_spans=False, + ), + ], + ) + + worker = Worker( + client, + task_queue="otel-task-queue", + workflows=[OtelBasicWorkflow, OtelDirectApiWorkflow], + activities=[get_weather], + # CRITICAL: Sandbox passthrough required for direct OTEL API usage + # If you only use automatic instrumentation (OtelBasicWorkflow), + # this configuration is not required + workflow_runner=SandboxedWorkflowRunner( + SandboxRestrictions.default.with_passthrough_modules("opentelemetry") + ), + ) + + print("Starting OTEL tracing worker...") + print("- Task queue: otel-task-queue") + print("- OTLP endpoint: http://localhost:4317") + print("- Workflows: OtelBasicWorkflow, OtelDirectApiWorkflow") + print("\nConfiguration:") + print(" - Automatic instrumentation: ENABLED (all workflows)") + print(" - Direct OTEL API support: ENABLED (sandbox passthrough configured)") + print(" - Temporal spans: DISABLED (add_temporal_spans=False)") + print("\nView traces at: http://localhost:3000/explore (Grafana Tempo)") + print(" or http://localhost:16686/ (Jaeger)\n") + + await worker.run() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/openai_agents/otel_tracing/workflows/__init__.py b/openai_agents/otel_tracing/workflows/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openai_agents/otel_tracing/workflows/otel_basic_workflow.py b/openai_agents/otel_tracing/workflows/otel_basic_workflow.py new file mode 100644 index 00000000..f5af8aa0 --- /dev/null +++ b/openai_agents/otel_tracing/workflows/otel_basic_workflow.py @@ -0,0 +1,58 @@ +"""Basic OTEL tracing workflow demonstrating automatic instrumentation. + +This workflow shows the simplest OTEL integration - just configure exporters in the +plugin and all agent/model/activity spans are automatically instrumented. +""" + +from dataclasses import dataclass +from datetime import timedelta + +from agents import Agent, Runner +from temporalio import activity, workflow +from temporalio.contrib import openai_agents as temporal_agents + + +@dataclass +class Weather: + city: str + temperature_range: str + conditions: str + + +@activity.defn +async def get_weather(city: str) -> str: + """Get the weather for a given city.""" + weather = Weather( + city=city, temperature_range="14-20C", conditions="Sunny with wind." + ) + return f"{weather.city}: {weather.conditions}, {weather.temperature_range}" + + +@workflow.defn +class OtelBasicWorkflow: + """Workflow demonstrating automatic OTEL instrumentation. + + The OTEL integration automatically creates spans for: + - Workflow execution + - Agent runs + - Model invocations (as activities) + - Tool/activity calls + + No manual span creation needed! + """ + + @workflow.run + async def run(self, question: str) -> str: + agent = Agent( + name="Weather Assistant", + instructions="You are a helpful weather assistant.", + tools=[ + temporal_agents.workflow.activity_as_tool( + get_weather, start_to_close_timeout=timedelta(seconds=10) + ) + ], + ) + + # All spans are automatically created - no manual instrumentation required! + result = await Runner.run(agent, input=question) + return result.final_output diff --git a/openai_agents/otel_tracing/workflows/otel_custom_spans_workflow.py b/openai_agents/otel_tracing/workflows/otel_custom_spans_workflow.py new file mode 100644 index 00000000..639a05e2 --- /dev/null +++ b/openai_agents/otel_tracing/workflows/otel_custom_spans_workflow.py @@ -0,0 +1,78 @@ +"""Custom spans workflow demonstrating logical grouping with trace() + custom_span(). + +This workflow shows how to use trace() wrapper with custom_span() to create logical +groupings of related operations, while still benefiting from automatic instrumentation +of agent/model/activity calls. + +IMPORTANT: When using custom_span(), wrap it with trace() in the workflow (not client). +""" + +from dataclasses import dataclass +from datetime import timedelta + +from agents import Agent, Runner, custom_span, trace +from temporalio import activity, workflow +from temporalio.contrib import openai_agents as temporal_agents + + +@dataclass +class Weather: + city: str + temperature_range: str + conditions: str + + +@activity.defn +async def get_weather(city: str) -> str: + """Get the weather for a given city.""" + weather = Weather( + city=city, temperature_range="14-20C", conditions="Sunny with wind." + ) + return f"{weather.city}: {weather.conditions}, {weather.temperature_range}" + + +@workflow.defn +class OtelCustomSpansWorkflow: + """Workflow demonstrating custom spans for logical grouping. + + This example shows how to use trace() + custom_span() to create logical + groupings of related operations. This pattern is useful when you want to: + - Group related operations under a single span + - Add meaningful structure to your traces + - Keep instrumentation simple while adding context + + IMPORTANT: When using custom_span(), you must wrap it with trace() in the + workflow to establish proper trace context. Never use trace() in client code. + + The OTEL integration still automatically creates spans for: + - Workflow execution + - Agent runs + - Model invocations (as activities) + - Tool/activity calls + """ + + @workflow.run + async def run(self) -> str: + with trace("Custom span sample"): + agent = Agent( + name="Weather Assistant", + instructions="You are a helpful weather assistant. Be concise.", + tools=[ + temporal_agents.workflow.activity_as_tool( + get_weather, start_to_close_timeout=timedelta(seconds=10) + ) + ], + ) + + # Use custom_span to group multiple related agent calls under one logical operation + # This makes it easy to see all weather checks for this request in your trace + with custom_span("Multi-city weather check"): + cities = ["Tokyo", "Paris", "New York"] + results = [] + for city in cities: + result = await Runner.run( + agent, input=f"What's the weather in {city}?" + ) + results.append(f"{city}: {result.final_output}") + + return "\n\n".join(results) diff --git a/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py b/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py new file mode 100644 index 00000000..72cfb9f7 --- /dev/null +++ b/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py @@ -0,0 +1,128 @@ +"""Direct OTEL API usage workflow demonstrating custom instrumentation. + +This workflow shows how to use the OpenTelemetry API directly in workflows +to instrument custom business logic, add domain-specific spans, and set +custom attributes. + +CRITICAL REQUIREMENTS: +1. Wrap direct OTEL calls in custom_span() from Agents SDK +2. Configure sandbox passthrough for opentelemetry module +""" + +from dataclasses import dataclass +from datetime import timedelta + +import opentelemetry.trace +from agents import Agent, Runner, custom_span +from temporalio import activity, workflow +from temporalio.contrib import openai_agents as temporal_agents + + +@dataclass +class Weather: + city: str + temperature_range: str + conditions: str + air_quality: str = "Good" + + +@activity.defn +async def get_weather(city: str) -> str: + """Get the weather for a given city.""" + weather = Weather( + city=city, + temperature_range="14-20C", + conditions="Sunny with wind.", + air_quality="Good", + ) + return f"{weather.city}: {weather.conditions}, {weather.temperature_range}, Air Quality: {weather.air_quality}" + + +def validate_city_name(city: str) -> bool: + """Validate that city name is reasonable.""" + # Simple validation logic + return len(city) > 0 and len(city) < 100 and city.replace(" ", "").isalpha() + + +def calculate_travel_score(weather: str) -> int: + """Calculate a travel score based on weather conditions.""" + # Simple scoring logic + score = 50 + if "sunny" in weather.lower(): + score += 30 + if "wind" in weather.lower(): + score += 10 + if "good" in weather.lower(): + score += 10 + return score + + +@workflow.defn +class OtelDirectApiWorkflow: + """Workflow demonstrating direct OTEL API usage for custom instrumentation. + + This workflow shows practical use cases for direct OTEL API: + - Instrumenting business logic validation + - Adding domain-specific spans + - Setting custom attributes for observability + - Creating logical groupings of operations + + IMPORTANT: Direct OTEL API calls MUST be wrapped in custom_span() from + the Agents SDK to establish the bridge between SDK trace context and OTEL. + """ + + @workflow.run + async def run(self, city: str) -> str: + # custom_span() establishes OTEL context bridge + # Direct OTEL API calls within this block will be properly parented + with custom_span("Travel recommendation workflow"): + tracer = opentelemetry.trace.get_tracer(__name__) + + # Custom instrumentation: validate input + with tracer.start_as_current_span("validate-input") as span: + span.set_attribute("input.city", city) + is_valid = validate_city_name(city) + span.set_attribute( + "validation.result", "valid" if is_valid else "invalid" + ) + + if not is_valid: + span.set_attribute("error", "Invalid city name") + return "Invalid city name provided" + + # Agent execution with automatic instrumentation + agent = Agent( + name="Travel Weather Assistant", + instructions="You are a helpful travel weather assistant. Provide weather information in a friendly way.", + tools=[ + temporal_agents.workflow.activity_as_tool( + get_weather, start_to_close_timeout=timedelta(seconds=10) + ) + ], + ) + + with tracer.start_as_current_span("fetch-weather-info") as span: + span.set_attribute("request.city", city) + result = await Runner.run( + agent, input=f"What's the weather like in {city}?" + ) + weather_info = result.final_output + span.set_attribute("response.length", len(weather_info)) + + # Custom instrumentation: calculate business metric + with tracer.start_as_current_span("calculate-travel-score") as span: + span.set_attribute("city", city) + travel_score = calculate_travel_score(weather_info) + span.set_attribute("travel.score", travel_score) + span.set_attribute( + "travel.recommendation", + "recommended" if travel_score > 70 else "not_recommended", + ) + + # Custom instrumentation: format final response + with tracer.start_as_current_span("format-response") as span: + span.set_attribute("include.score", True) + final_response = f"{weather_info}\n\nTravel Score: {travel_score}/100" + span.set_attribute("response.final_length", len(final_response)) + + return final_response diff --git a/pyproject.toml b/pyproject.toml index bf97ef04..9553e41d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,9 @@ open-telemetry = [ ] openai-agents = [ "openai-agents[litellm] == 0.3.2", - "temporalio[openai-agents] >= 1.18.0", + "temporalio[openai-agents,opentelemetry] >= 1.18.0", + "openinference-instrumentation-openai-agents>=0.1.0", + "opentelemetry-exporter-otlp-proto-grpc", "requests>=2.32.0,<3", ] pydantic-converter = ["pydantic>=2.10.6,<3"] From 71c27baf69c86e9d1471b2a93194aa8a1271d0b6 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Tue, 3 Feb 2026 12:59:16 -0800 Subject: [PATCH 2/3] Update OTEL examples with correct trace() usage pattern Corrects the documentation and code to reflect the proper usage of trace(): - trace() should be used in workflows when using custom_span() (patterns 2 & 3) - Never use trace() in client code - Pattern 1 (basic): No trace() needed - plugin creates root trace automatically - Pattern 2 (custom spans): trace() + custom_span() in workflow - Pattern 3 (direct API): trace() + custom_span() + OTEL tracer in workflow Updates: - README with clearer pattern explanations and examples - Direct API workflow to use trace() + custom_span() pattern - All docstrings to document the correct patterns - Troubleshooting section with pattern-specific guidance Co-Authored-By: Claude Sonnet 4.5 --- openai_agents/otel_tracing/README.md | 315 ++++++++++-------- openai_agents/otel_tracing/run_otel_basic.py | 1 - .../otel_tracing/run_otel_direct_api.py | 10 +- openai_agents/otel_tracing/run_worker.py | 10 +- .../workflows/otel_basic_workflow.py | 6 +- .../workflows/otel_direct_api_workflow.py | 118 ++++--- 6 files changed, 250 insertions(+), 210 deletions(-) diff --git a/openai_agents/otel_tracing/README.md b/openai_agents/otel_tracing/README.md index 4630f2cd..d236e36a 100644 --- a/openai_agents/otel_tracing/README.md +++ b/openai_agents/otel_tracing/README.md @@ -1,48 +1,17 @@ -# OpenTelemetry (OTEL) Tracing for OpenAI Agents +# OpenTelemetry (OTEL) Tracing -This example demonstrates how to instrument OpenAI Agents workflows with OpenTelemetry (OTEL) for distributed tracing and observability. +Examples demonstrating OpenTelemetry tracing integration for OpenAI Agents workflows. -Traces can be exported to any OTEL-compatible backend such as Jaeger, Grafana Tempo, Datadog, New Relic, or other tracing systems. +*For background on OpenTelemetry integration, see the [SDK documentation](https://github.com/temporalio/sdk-python/blob/main/temporalio/contrib/openai_agents/README.md#opentelemetry-integration).* -## Overview +This example shows three progressive patterns: +1. **Basic**: Pure automatic instrumentation - plugin handles everything +2. **Custom Spans**: Automatic instrumentation + `custom_span()` for logical grouping +3. **Direct API**: `custom_span()` + direct OpenTelemetry API for detailed instrumentation -The Temporal OpenAI Agents SDK provides built-in OTEL integration that: -- **Automatically instruments** agent runs, model calls, and activities -- **Is replay-safe** - spans are only exported when workflows actually complete (not during replay) -- **Provides deterministic IDs** - consistent span/trace IDs across workflow replays -- **Supports multiple exporters** - send traces to multiple backends simultaneously +## Prerequisites -## Two Instrumentation Patterns - -### 1. Automatic Instrumentation (Recommended for Most Users) - -The SDK automatically creates spans for: -- Agent execution -- Model invocations (as Temporal activities) -- Tool/activity calls -- Workflow lifecycle events - -**Use this when:** You want visibility into agent behavior without custom instrumentation. - -See `run_otel_basic.py` for an example. - -### 2. Direct OTEL API Usage (Advanced) - -You can use the OpenTelemetry API directly in workflows to: -- Instrument custom business logic -- Add domain-specific spans and attributes -- Integrate with organization-wide OTEL conventions -- Monitor performance of specific operations - -**Use this when:** You need to trace custom workflow logic beyond agent/model calls. - -See `run_otel_direct_api.py` for an example. - -## Quick Start - -### Prerequisites - -Ensure you have an OTEL collector or backend running. For local testing with Grafana Tempo: +You need an OTEL-compatible backend running locally. For quick setup with Grafana Tempo: ```bash git clone https://github.com/grafana/tempo.git @@ -53,185 +22,239 @@ docker compose up -d View traces at: http://localhost:3000/explore -### Install Dependencies - -```bash -uv sync -``` +Alternatively, use Jaeger at http://localhost:16686/ -### Start the Worker +## Running the Examples +First, start the worker: ```bash uv run openai_agents/otel_tracing/run_worker.py ``` -### Run Examples - -In separate terminals: +Then run examples in separate terminals: -**Basic automatic instrumentation:** +### 1. Basic Example - Pure Automatic Instrumentation +Shows automatic tracing without any manual code: ```bash uv run openai_agents/otel_tracing/run_otel_basic.py ``` -**Direct OTEL API usage:** +### 2. Custom Spans Example - Logical Grouping +Shows using `custom_span()` to group related operations: +```bash +uv run openai_agents/otel_tracing/run_otel_custom_spans.py +``` + +### 3. Direct API Example - Detailed Custom Instrumentation +Shows using direct OpenTelemetry API for fine-grained custom instrumentation: ```bash uv run openai_agents/otel_tracing/run_otel_direct_api.py ``` -## Implementation Details +## Example Progression -### Plugin Configuration +The three examples show increasing levels of instrumentation: -Configure OTEL exporters in the `OpenAIAgentsPlugin`: +| Example | Manual Code | Use Case | +|---------|-------------|----------| +| **1. Basic** | None | Just want automatic tracing | +| **2. Custom Spans** | `custom_span()` | Group related operations logically | +| **3. Direct API** | `custom_span()` + OTEL tracer | Add detailed spans with custom attributes | + +## What Gets Traced + +The integration automatically creates spans for: +- Agent execution +- Model invocations (as Temporal activities) +- Tool/activity calls +- Workflow lifecycle events (optional) + +You can add custom instrumentation using three patterns: +1. **Pure Automatic** (example 1): No code needed - plugin handles everything +2. **Custom Spans** (example 2): `trace()` + `custom_span()` from Agents SDK for logical grouping +3. **Direct OTEL API** (example 3): `trace()` + `custom_span()` + OTEL tracer for detailed spans with attributes + +**Key Rule**: Never use `trace()` in client code. Only use it inside workflows when you need `custom_span()` (patterns 2 and 3). + +## Key Configuration + +### Plugin Setup (Worker & Client) ```python from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from temporalio.contrib.openai_agents import OpenAIAgentsPlugin +from temporalio.contrib.openai_agents import OpenAIAgentsPlugin, ModelActivityParameters + +exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) client = await Client.connect( "localhost:7233", plugins=[ OpenAIAgentsPlugin( model_params=ModelActivityParameters( - start_to_close_timeout=timedelta(seconds=30) + start_to_close_timeout=timedelta(seconds=60) ), - otel_exporters=[ - OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) - ], - add_temporal_spans=False, # Optional: exclude Temporal internal spans + otel_exporters=[exporter], # Enable OTEL export + add_temporal_spans=False, # Optional: exclude Temporal internal spans ), ], ) ``` -### Key Parameters +### Exporters -**`otel_exporters`**: List of OTEL span exporters -- `OTLPSpanExporter` - OTLP protocol (most common) -- `InMemorySpanExporter` - For testing -- `ConsoleSpanExporter` - Debug output -- Multiple exporters supported simultaneously +Common OTEL exporters: +- `OTLPSpanExporter` - For Grafana Tempo, Jaeger, and most OTEL backends +- `ConsoleSpanExporter` - For debugging (prints to console) +- Multiple exporters can be used simultaneously -**`add_temporal_spans`**: Whether to include Temporal internal spans (default: `True`) -- `False` - Cleaner traces focused on agent logic -- `True` - Full visibility including Temporal internals (startWorkflow, executeActivity, etc.) +### Environment Variables -### Direct OTEL API Usage +Optionally set the service name: +```bash +export OTEL_SERVICE_NAME=my-agent-service +``` -**CRITICAL REQUIREMENTS** for using `opentelemetry.trace` API directly in workflows: +## Understanding Trace Context Patterns -#### 1. Wrap in `custom_span()` from Agents SDK +The integration supports three patterns depending on your instrumentation needs: -Direct OTEL calls **MUST** be wrapped in `custom_span()` to establish the bridge between Agent SDK trace context and OTEL context: +### Pattern 1: Pure Automatic Instrumentation (Basic Example) +No manual code - plugin creates root trace automatically: ```python -from agents import custom_span +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self): + # No trace(), no custom_span() needed + # Plugin automatically creates root trace and all spans + result = await Runner.run(agent, input=question) + return result +``` + +### Pattern 2: Logical Grouping with Custom Spans (Custom Spans Example) +Use `trace()` in workflow + `custom_span()` for logical grouping: + +```python +from agents import trace, custom_span + +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self): + # trace() in workflow establishes context for custom_span() + with trace("My workflow"): + with custom_span("Multi-city check"): + # Group related operations + for city in cities: + result = await Runner.run(agent, input=f"Check {city}") + return result +``` + +**IMPORTANT**: When using `custom_span()`, you must wrap it with `trace()` in the workflow. Never use `trace()` in client code - only in workflows. + +### Pattern 3: Direct OTEL API (Direct API Example) +Use `trace()` + `custom_span()` wrapper + direct OpenTelemetry API for detailed instrumentation: + +```python +from agents import trace, custom_span import opentelemetry.trace @workflow.defn class MyWorkflow: @workflow.run async def run(self): - # ✅ CORRECT - custom_span establishes OTEL context - with custom_span("My workflow logic"): - tracer = opentelemetry.trace.get_tracer(__name__) - with tracer.start_as_current_span("custom-instrumentation") as span: - span.set_attribute("business.metric", 42) - # This span will be properly parented - result = await my_business_logic() - - # ❌ WRONG - becomes orphaned root span, disconnected from trace - tracer = opentelemetry.trace.get_tracer(__name__) - with tracer.start_as_current_span("orphaned-span"): - # No connection to the agent trace! - pass + # trace() establishes root context, custom_span() bridges to OTEL + with trace("My workflow"): + with custom_span("My workflow logic"): + tracer = opentelemetry.trace.get_tracer(__name__) + + with tracer.start_as_current_span("Data processing") as span: + span.set_attribute("my.attribute", "value") + data = await self.process_data() + + with tracer.start_as_current_span("Business logic") as span: + result = await self.execute_business_logic(data) + return result ``` -#### 2. Configure Sandbox Passthrough +**Why both are required**: When using `custom_span()`, you must wrap it with `trace()` in the workflow. The `custom_span()` then bridges to OpenTelemetry's context system for direct API calls. -The `opentelemetry` module must be allowed in the workflow sandbox: +### Worker Configuration for Direct OTEL API + +When using direct OTEL API (Pattern 3), configure sandbox passthrough: ```python from temporalio.worker import Worker -from temporalio.worker.workflow_sandbox import ( - SandboxedWorkflowRunner, - SandboxRestrictions, -) +from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner, SandboxRestrictions worker = Worker( client, - task_queue="otel-task-queue", + task_queue="my-queue", workflows=[MyWorkflow], + # Required ONLY for Pattern 3 (direct OTEL API usage) workflow_runner=SandboxedWorkflowRunner( SandboxRestrictions.default.with_passthrough_modules("opentelemetry") ), ) ``` -### Trace Context Propagation +**Note**: Patterns 1 and 2 (automatic and custom_span only) don't require sandbox configuration. -Traces automatically propagate through the system: - -``` -Client trace - └─> Workflow execution - ├─> Agent span - │ └─> Model activity - ├─> Custom span (if using direct API) - └─> Tool activity -``` +## Troubleshooting -- **Client → Workflow**: Trace context propagates when starting workflow within `trace()` block -- **Workflow → Activity**: Context automatically propagates to activities -- **Replay-safe**: Spans only export on actual completion, not during replay +### Multiple separate traces instead of one unified trace -### Environment Configuration +**For Custom Spans (Pattern 2)**: Ensure you wrap `custom_span()` with `trace()` in the workflow: +```python +from agents import trace, custom_span -Set the OTEL service name (optional): -```bash -export OTEL_SERVICE_NAME=my-agent-service +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self): + # ✅ CORRECT - trace() wraps custom_span() + with trace("My workflow"): + with custom_span("My grouping"): + # Related operations + pass ``` -## Common Use Cases - -### Basic Monitoring -Use automatic instrumentation to: -- Monitor agent performance -- Debug agent behavior -- Track model API usage -- Identify bottlenecks - -### Custom Business Logic -Use direct OTEL API to: -- Instrument domain-specific operations -- Add business metrics as span attributes -- Create logical groupings of related operations -- Integrate with existing observability stack +**For Direct OTEL API (Pattern 3)**: Ensure workflow wraps all direct OTEL calls in `custom_span()`: +```python +from agents import custom_span +import opentelemetry.trace -### Production Observability -- Export to multiple backends (e.g., Jaeger for dev, Datadog for prod) -- Use `add_temporal_spans=False` for cleaner production traces -- Add custom attributes for filtering/grouping in your observability tool +@workflow.defn +class MyWorkflow: + @workflow.run + async def run(self): + # ✅ CORRECT - All direct OTEL spans inside custom_span() + with custom_span("My workflow"): + tracer = opentelemetry.trace.get_tracer(__name__) + with tracer.start_as_current_span("span1"): + pass + with tracer.start_as_current_span("span2"): + pass +``` -## Troubleshooting +**NEVER use `trace()` in client code** - this creates disconnected traces. Only use `trace()` inside workflows. ### Spans not appearing in backend -- Verify OTLP endpoint is accessible -- Check that backend is configured to accept OTLP +- Verify OTLP endpoint is accessible: `http://localhost:4317` +- Check backend is running: `docker compose ps` - Ensure workflow completes (spans only export on completion) -### Direct OTEL spans become root spans -- Verify you're wrapping calls in `custom_span()` -- Check sandbox passthrough is configured -- Ensure you're within an existing trace context +### Direct OTEL spans are orphaned +- **For Pattern 2 (custom_span)**: Ensure you use `trace()` wrapper in workflow +- **For Pattern 3 (direct OTEL)**: Verify workflow wraps ALL direct OTEL calls in `custom_span()` +- Check sandbox passthrough is configured for `opentelemetry` module (Pattern 3 only) -### Duplicate spans across replays -- This is expected behavior during development with workflow cache -- Spans are only exported once per actual execution, not per replay +## Dependencies -## Related Examples - -- [grafana-tempo-openai-example](../../../../grafana-tempo-openai-example/) - End-to-end observability demo -- [basic/](../basic/) - Simple agent examples without OTEL -- [financial_research_agent](../financial_research_agent/) - Complex multi-agent example +Required packages (already in `openai-agents` dependency group): +```toml +temporalio[openai-agents,opentelemetry] +openinference-instrumentation-openai-agents +opentelemetry-exporter-otlp-proto-grpc +``` diff --git a/openai_agents/otel_tracing/run_otel_basic.py b/openai_agents/otel_tracing/run_otel_basic.py index c45452f0..b7bdef44 100755 --- a/openai_agents/otel_tracing/run_otel_basic.py +++ b/openai_agents/otel_tracing/run_otel_basic.py @@ -52,7 +52,6 @@ async def main(): print(" - Grafana Tempo: http://localhost:3000/explore") print(" - Jaeger: http://localhost:16686/") print("\nExpected spans in trace:") - print(" - Workflow execution") print(" - Agent run (Weather Assistant)") print(" - Model invocation (activity)") print(" - Tool call (get_weather activity)") diff --git a/openai_agents/otel_tracing/run_otel_direct_api.py b/openai_agents/otel_tracing/run_otel_direct_api.py index 1e3a6873..0dc46bb5 100755 --- a/openai_agents/otel_tracing/run_otel_direct_api.py +++ b/openai_agents/otel_tracing/run_otel_direct_api.py @@ -5,8 +5,9 @@ instrument custom business logic, add domain-specific spans, and set custom attributes. -The workflow uses custom_span() to establish OTEL context, then creates -custom spans for validation, business logic, and formatting operations. +The workflow uses trace() + custom_span() to establish OTEL context, then creates +custom spans for validation, business logic, and formatting operations using the +direct OpenTelemetry tracer API. """ import asyncio @@ -42,6 +43,8 @@ async def main(): city = "Paris" print(f"Getting travel recommendation for: {city}\n") + # The plugin automatically creates the root trace context. + # The workflow uses custom_span() to bridge to OpenTelemetry context for direct API usage. result = await client.execute_workflow( OtelDirectApiWorkflow.run, city, @@ -55,7 +58,8 @@ async def main(): print(" - Grafana Tempo: http://localhost:3000/explore") print(" - Jaeger: http://localhost:16686/") print("\nExpected spans in trace:") - print(" - Travel recommendation workflow (custom_span)") + print(" - Travel recommendation workflow (trace)") + print(" - Travel recommendation processing (custom_span)") print(" - validate-input (direct OTEL span)") print(" - Agent run (Travel Weather Assistant)") print(" - fetch-weather-info (direct OTEL span)") diff --git a/openai_agents/otel_tracing/run_worker.py b/openai_agents/otel_tracing/run_worker.py index f2410618..8ef139e5 100755 --- a/openai_agents/otel_tracing/run_worker.py +++ b/openai_agents/otel_tracing/run_worker.py @@ -26,6 +26,9 @@ OtelBasicWorkflow, get_weather, ) +from openai_agents.otel_tracing.workflows.otel_custom_spans_workflow import ( + OtelCustomSpansWorkflow, +) from openai_agents.otel_tracing.workflows.otel_direct_api_workflow import ( OtelDirectApiWorkflow, ) @@ -54,7 +57,7 @@ async def main(): worker = Worker( client, task_queue="otel-task-queue", - workflows=[OtelBasicWorkflow, OtelDirectApiWorkflow], + workflows=[OtelBasicWorkflow, OtelCustomSpansWorkflow, OtelDirectApiWorkflow], activities=[get_weather], # CRITICAL: Sandbox passthrough required for direct OTEL API usage # If you only use automatic instrumentation (OtelBasicWorkflow), @@ -67,9 +70,12 @@ async def main(): print("Starting OTEL tracing worker...") print("- Task queue: otel-task-queue") print("- OTLP endpoint: http://localhost:4317") - print("- Workflows: OtelBasicWorkflow, OtelDirectApiWorkflow") + print( + "- Workflows: OtelBasicWorkflow, OtelCustomSpansWorkflow, OtelDirectApiWorkflow" + ) print("\nConfiguration:") print(" - Automatic instrumentation: ENABLED (all workflows)") + print(" - Custom spans support: ENABLED") print(" - Direct OTEL API support: ENABLED (sandbox passthrough configured)") print(" - Temporal spans: DISABLED (add_temporal_spans=False)") print("\nView traces at: http://localhost:3000/explore (Grafana Tempo)") diff --git a/openai_agents/otel_tracing/workflows/otel_basic_workflow.py b/openai_agents/otel_tracing/workflows/otel_basic_workflow.py index f5af8aa0..97ff9530 100644 --- a/openai_agents/otel_tracing/workflows/otel_basic_workflow.py +++ b/openai_agents/otel_tracing/workflows/otel_basic_workflow.py @@ -1,7 +1,7 @@ """Basic OTEL tracing workflow demonstrating automatic instrumentation. -This workflow shows the simplest OTEL integration - just configure exporters in the -plugin and all agent/model/activity spans are automatically instrumented. +This workflow shows pure automatic instrumentation - the plugin handles all trace +creation and span instrumentation without any manual code. """ from dataclasses import dataclass @@ -38,7 +38,7 @@ class OtelBasicWorkflow: - Model invocations (as activities) - Tool/activity calls - No manual span creation needed! + No manual instrumentation needed - just configure the plugin! """ @workflow.run diff --git a/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py b/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py index 72cfb9f7..6a869823 100644 --- a/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py +++ b/openai_agents/otel_tracing/workflows/otel_direct_api_workflow.py @@ -5,15 +5,18 @@ custom attributes. CRITICAL REQUIREMENTS: -1. Wrap direct OTEL calls in custom_span() from Agents SDK -2. Configure sandbox passthrough for opentelemetry module +1. Use trace() wrapper with custom_span() from Agents SDK to establish context +2. Wrap direct OTEL tracer calls in custom_span() (establishes OTEL bridge) +3. Configure sandbox passthrough for opentelemetry module in worker + +Pattern: trace() -> custom_span() -> tracer.start_as_current_span() """ from dataclasses import dataclass from datetime import timedelta import opentelemetry.trace -from agents import Agent, Runner, custom_span +from agents import Agent, Runner, custom_span, trace from temporalio import activity, workflow from temporalio.contrib import openai_agents as temporal_agents @@ -63,66 +66,71 @@ class OtelDirectApiWorkflow: This workflow shows practical use cases for direct OTEL API: - Instrumenting business logic validation - - Adding domain-specific spans + - Adding domain-specific spans with custom attributes - Setting custom attributes for observability - - Creating logical groupings of operations + - Creating detailed traces with business metrics - IMPORTANT: Direct OTEL API calls MUST be wrapped in custom_span() from - the Agents SDK to establish the bridge between SDK trace context and OTEL. + IMPORTANT: When using direct OTEL API, wrap everything in trace() + custom_span(): + - trace() establishes the root trace context (required when using custom_span) + - custom_span() bridges to OpenTelemetry context for direct tracer calls + - Direct OTEL spans (tracer.start_as_current_span) go inside custom_span() """ @workflow.run async def run(self, city: str) -> str: - # custom_span() establishes OTEL context bridge - # Direct OTEL API calls within this block will be properly parented - with custom_span("Travel recommendation workflow"): - tracer = opentelemetry.trace.get_tracer(__name__) - - # Custom instrumentation: validate input - with tracer.start_as_current_span("validate-input") as span: - span.set_attribute("input.city", city) - is_valid = validate_city_name(city) - span.set_attribute( - "validation.result", "valid" if is_valid else "invalid" - ) - - if not is_valid: - span.set_attribute("error", "Invalid city name") - return "Invalid city name provided" - - # Agent execution with automatic instrumentation - agent = Agent( - name="Travel Weather Assistant", - instructions="You are a helpful travel weather assistant. Provide weather information in a friendly way.", - tools=[ - temporal_agents.workflow.activity_as_tool( - get_weather, start_to_close_timeout=timedelta(seconds=10) + # trace() establishes the root context needed for custom_span() and direct OTEL API + with trace("Travel recommendation workflow"): + # custom_span() establishes OTEL context bridge for direct OTEL API calls + with custom_span("Travel recommendation processing"): + tracer = opentelemetry.trace.get_tracer(__name__) + + # Custom instrumentation: validate input + with tracer.start_as_current_span("validate-input") as span: + span.set_attribute("input.city", city) + is_valid = validate_city_name(city) + span.set_attribute( + "validation.result", "valid" if is_valid else "invalid" ) - ], - ) - with tracer.start_as_current_span("fetch-weather-info") as span: - span.set_attribute("request.city", city) - result = await Runner.run( - agent, input=f"What's the weather like in {city}?" - ) - weather_info = result.final_output - span.set_attribute("response.length", len(weather_info)) - - # Custom instrumentation: calculate business metric - with tracer.start_as_current_span("calculate-travel-score") as span: - span.set_attribute("city", city) - travel_score = calculate_travel_score(weather_info) - span.set_attribute("travel.score", travel_score) - span.set_attribute( - "travel.recommendation", - "recommended" if travel_score > 70 else "not_recommended", + if not is_valid: + span.set_attribute("error", "Invalid city name") + return "Invalid city name provided" + + # Agent execution with automatic instrumentation + agent = Agent( + name="Travel Weather Assistant", + instructions="You are a helpful travel weather assistant. Provide weather information in a friendly way.", + tools=[ + temporal_agents.workflow.activity_as_tool( + get_weather, start_to_close_timeout=timedelta(seconds=10) + ) + ], ) - # Custom instrumentation: format final response - with tracer.start_as_current_span("format-response") as span: - span.set_attribute("include.score", True) - final_response = f"{weather_info}\n\nTravel Score: {travel_score}/100" - span.set_attribute("response.final_length", len(final_response)) + with tracer.start_as_current_span("fetch-weather-info") as span: + span.set_attribute("request.city", city) + result = await Runner.run( + agent, input=f"What's the weather like in {city}?" + ) + weather_info = result.final_output + span.set_attribute("response.length", len(weather_info)) + + # Custom instrumentation: calculate business metric + with tracer.start_as_current_span("calculate-travel-score") as span: + span.set_attribute("city", city) + travel_score = calculate_travel_score(weather_info) + span.set_attribute("travel.score", travel_score) + span.set_attribute( + "travel.recommendation", + "recommended" if travel_score > 70 else "not_recommended", + ) + + # Custom instrumentation: format final response + with tracer.start_as_current_span("format-response") as span: + span.set_attribute("include.score", True) + final_response = ( + f"{weather_info}\n\nTravel Score: {travel_score}/100" + ) + span.set_attribute("response.final_length", len(final_response)) - return final_response + return final_response From 788a5a5c1d3a1c3019c0e1e0c8e9a9d4e216fbb5 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Tue, 3 Feb 2026 13:13:53 -0800 Subject: [PATCH 3/3] Align OTEL tracing samples with repository style conventions Rename client scripts to use _workflow.py suffix for consistency with other samples. Simplify code by removing verbose docstrings, comments, and output messages to match the minimal style used throughout the repository. Co-Authored-By: Claude Sonnet 4.5 --- openai_agents/otel_tracing/README.md | 6 +- ...el_basic.py => run_otel_basic_workflow.py} | 26 +------ ...s.py => run_otel_custom_spans_workflow.py} | 21 +---- .../otel_tracing/run_otel_direct_api.py | 77 ------------------- .../run_otel_direct_api_workflow.py | 41 ++++++++++ openai_agents/otel_tracing/run_worker.py | 33 -------- 6 files changed, 47 insertions(+), 157 deletions(-) rename openai_agents/otel_tracing/{run_otel_basic.py => run_otel_basic_workflow.py} (55%) mode change 100755 => 100644 rename openai_agents/otel_tracing/{run_otel_custom_spans.py => run_otel_custom_spans_workflow.py} (57%) mode change 100755 => 100644 delete mode 100755 openai_agents/otel_tracing/run_otel_direct_api.py create mode 100644 openai_agents/otel_tracing/run_otel_direct_api_workflow.py diff --git a/openai_agents/otel_tracing/README.md b/openai_agents/otel_tracing/README.md index d236e36a..701d0e99 100644 --- a/openai_agents/otel_tracing/README.md +++ b/openai_agents/otel_tracing/README.md @@ -36,19 +36,19 @@ Then run examples in separate terminals: ### 1. Basic Example - Pure Automatic Instrumentation Shows automatic tracing without any manual code: ```bash -uv run openai_agents/otel_tracing/run_otel_basic.py +uv run openai_agents/otel_tracing/run_otel_basic_workflow.py ``` ### 2. Custom Spans Example - Logical Grouping Shows using `custom_span()` to group related operations: ```bash -uv run openai_agents/otel_tracing/run_otel_custom_spans.py +uv run openai_agents/otel_tracing/run_otel_custom_spans_workflow.py ``` ### 3. Direct API Example - Detailed Custom Instrumentation Shows using direct OpenTelemetry API for fine-grained custom instrumentation: ```bash -uv run openai_agents/otel_tracing/run_otel_direct_api.py +uv run openai_agents/otel_tracing/run_otel_direct_api_workflow.py ``` ## Example Progression diff --git a/openai_agents/otel_tracing/run_otel_basic.py b/openai_agents/otel_tracing/run_otel_basic_workflow.py old mode 100755 new mode 100644 similarity index 55% rename from openai_agents/otel_tracing/run_otel_basic.py rename to openai_agents/otel_tracing/run_otel_basic_workflow.py index b7bdef44..27480b3a --- a/openai_agents/otel_tracing/run_otel_basic.py +++ b/openai_agents/otel_tracing/run_otel_basic_workflow.py @@ -1,13 +1,3 @@ -#!/usr/bin/env python3 -"""Client for basic OTEL tracing example. - -This demonstrates the simplest OTEL integration - automatic instrumentation -of agent/model/activity spans without any custom code. - -The worker configuration handles all OTEL setup. This client just executes -the workflow normally. -""" - import asyncio import uuid from datetime import timedelta @@ -20,7 +10,6 @@ async def main(): - # Configure OTLP exporter (same as worker) exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) client = await Client.connect( @@ -36,25 +25,14 @@ async def main(): ], ) - question = "What's the weather like in Tokyo?" - print(f"Question: {question}\n") - result = await client.execute_workflow( OtelBasicWorkflow.run, - question, + "What's the weather like in Tokyo?", id=f"otel-basic-workflow-{uuid.uuid4()}", task_queue="otel-task-queue", ) - print(f"Answer: {result}\n") - print("✓ Workflow completed") - print("\nView traces at:") - print(" - Grafana Tempo: http://localhost:3000/explore") - print(" - Jaeger: http://localhost:16686/") - print("\nExpected spans in trace:") - print(" - Agent run (Weather Assistant)") - print(" - Model invocation (activity)") - print(" - Tool call (get_weather activity)") + print(f"Result: {result}") if __name__ == "__main__": diff --git a/openai_agents/otel_tracing/run_otel_custom_spans.py b/openai_agents/otel_tracing/run_otel_custom_spans_workflow.py old mode 100755 new mode 100644 similarity index 57% rename from openai_agents/otel_tracing/run_otel_custom_spans.py rename to openai_agents/otel_tracing/run_otel_custom_spans_workflow.py index 95d079a4..7ece84f4 --- a/openai_agents/otel_tracing/run_otel_custom_spans.py +++ b/openai_agents/otel_tracing/run_otel_custom_spans_workflow.py @@ -1,10 +1,3 @@ -#!/usr/bin/env python3 -"""Client for custom spans OTEL example. - -This demonstrates using custom_span() to create logical groupings in traces -while still benefiting from automatic instrumentation. -""" - import asyncio import uuid from datetime import timedelta @@ -19,7 +12,6 @@ async def main(): - # Configure OTLP exporter (same as worker) exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) client = await Client.connect( @@ -35,24 +27,13 @@ async def main(): ], ) - print("Checking weather for multiple cities...\n") - result = await client.execute_workflow( OtelCustomSpansWorkflow.run, id=f"otel-custom-spans-workflow-{uuid.uuid4()}", task_queue="otel-task-queue", ) - print(f"Results:\n{result}\n") - print("✓ Workflow completed") - print("\nView traces at:") - print(" - Grafana Tempo: http://localhost:3000/explore") - print(" - Jaeger: http://localhost:16686/") - print("\nExpected spans in trace:") - print(" - Multi-city weather check (custom_span grouping)") - print(" - Agent runs for Tokyo, Paris, New York (3 agents)") - print(" - Model invocations (activities)") - print(" - Tool calls (get_weather activities)") + print(f"Result:\n{result}") if __name__ == "__main__": diff --git a/openai_agents/otel_tracing/run_otel_direct_api.py b/openai_agents/otel_tracing/run_otel_direct_api.py deleted file mode 100755 index 0dc46bb5..00000000 --- a/openai_agents/otel_tracing/run_otel_direct_api.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -"""Client for direct OTEL API usage example. - -This demonstrates using the OpenTelemetry API directly in workflows to -instrument custom business logic, add domain-specific spans, and set -custom attributes. - -The workflow uses trace() + custom_span() to establish OTEL context, then creates -custom spans for validation, business logic, and formatting operations using the -direct OpenTelemetry tracer API. -""" - -import asyncio -import uuid -from datetime import timedelta - -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from temporalio.client import Client -from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin - -from openai_agents.otel_tracing.workflows.otel_direct_api_workflow import ( - OtelDirectApiWorkflow, -) - - -async def main(): - # Configure OTLP exporter (same as worker) - exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) - - client = await Client.connect( - "localhost:7233", - plugins=[ - OpenAIAgentsPlugin( - model_params=ModelActivityParameters( - start_to_close_timeout=timedelta(seconds=60) - ), - otel_exporters=[exporter], - add_temporal_spans=False, - ), - ], - ) - - city = "Paris" - print(f"Getting travel recommendation for: {city}\n") - - # The plugin automatically creates the root trace context. - # The workflow uses custom_span() to bridge to OpenTelemetry context for direct API usage. - result = await client.execute_workflow( - OtelDirectApiWorkflow.run, - city, - id=f"otel-direct-api-workflow-{uuid.uuid4()}", - task_queue="otel-task-queue", - ) - - print(f"Result:\n{result}\n") - print("✓ Workflow completed") - print("\nView traces at:") - print(" - Grafana Tempo: http://localhost:3000/explore") - print(" - Jaeger: http://localhost:16686/") - print("\nExpected spans in trace:") - print(" - Travel recommendation workflow (trace)") - print(" - Travel recommendation processing (custom_span)") - print(" - validate-input (direct OTEL span)") - print(" - Agent run (Travel Weather Assistant)") - print(" - fetch-weather-info (direct OTEL span)") - print(" - Model invocation (activity)") - print(" - Tool call (get_weather activity)") - print(" - calculate-travel-score (direct OTEL span)") - print(" - format-response (direct OTEL span)") - print("\nLook for custom attributes on spans:") - print(" - input.city, validation.result") - print(" - request.city, response.length") - print(" - travel.score, travel.recommendation") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/openai_agents/otel_tracing/run_otel_direct_api_workflow.py b/openai_agents/otel_tracing/run_otel_direct_api_workflow.py new file mode 100644 index 00000000..31151592 --- /dev/null +++ b/openai_agents/otel_tracing/run_otel_direct_api_workflow.py @@ -0,0 +1,41 @@ +import asyncio +import uuid +from datetime import timedelta + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from temporalio.client import Client +from temporalio.contrib.openai_agents import ModelActivityParameters, OpenAIAgentsPlugin + +from openai_agents.otel_tracing.workflows.otel_direct_api_workflow import ( + OtelDirectApiWorkflow, +) + + +async def main(): + exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) + + client = await Client.connect( + "localhost:7233", + plugins=[ + OpenAIAgentsPlugin( + model_params=ModelActivityParameters( + start_to_close_timeout=timedelta(seconds=60) + ), + otel_exporters=[exporter], + add_temporal_spans=False, + ), + ], + ) + + result = await client.execute_workflow( + OtelDirectApiWorkflow.run, + "Paris", + id=f"otel-direct-api-workflow-{uuid.uuid4()}", + task_queue="otel-task-queue", + ) + + print(f"Result:\n{result}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/openai_agents/otel_tracing/run_worker.py b/openai_agents/otel_tracing/run_worker.py index 8ef139e5..a6a5756d 100755 --- a/openai_agents/otel_tracing/run_worker.py +++ b/openai_agents/otel_tracing/run_worker.py @@ -1,15 +1,3 @@ -#!/usr/bin/env python3 -"""Worker for OTEL tracing examples. - -This worker demonstrates OTEL configuration for both automatic instrumentation -and direct OTEL API usage patterns. - -Configuration: -- OTLP exporter for sending traces to OTEL backend -- Sandbox passthrough for opentelemetry module (required for direct API usage) -- Optional add_temporal_spans parameter for controlling span verbosity -""" - import asyncio from datetime import timedelta @@ -35,9 +23,6 @@ async def main(): - # Configure OTLP exporter - # Default endpoint is localhost:4317 for Grafana Tempo/Jaeger - # Adjust endpoint for your OTEL backend (e.g., Datadog, New Relic, etc.) exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True) client = await Client.connect( @@ -48,7 +33,6 @@ async def main(): start_to_close_timeout=timedelta(seconds=60) ), otel_exporters=[exporter], - # Optional: Set to False to exclude Temporal internal spans for cleaner traces add_temporal_spans=False, ), ], @@ -59,28 +43,11 @@ async def main(): task_queue="otel-task-queue", workflows=[OtelBasicWorkflow, OtelCustomSpansWorkflow, OtelDirectApiWorkflow], activities=[get_weather], - # CRITICAL: Sandbox passthrough required for direct OTEL API usage - # If you only use automatic instrumentation (OtelBasicWorkflow), - # this configuration is not required workflow_runner=SandboxedWorkflowRunner( SandboxRestrictions.default.with_passthrough_modules("opentelemetry") ), ) - print("Starting OTEL tracing worker...") - print("- Task queue: otel-task-queue") - print("- OTLP endpoint: http://localhost:4317") - print( - "- Workflows: OtelBasicWorkflow, OtelCustomSpansWorkflow, OtelDirectApiWorkflow" - ) - print("\nConfiguration:") - print(" - Automatic instrumentation: ENABLED (all workflows)") - print(" - Custom spans support: ENABLED") - print(" - Direct OTEL API support: ENABLED (sandbox passthrough configured)") - print(" - Temporal spans: DISABLED (add_temporal_spans=False)") - print("\nView traces at: http://localhost:3000/explore (Grafana Tempo)") - print(" or http://localhost:16686/ (Jaeger)\n") - await worker.run()