Skip to content

Commit 8bb3d8e

Browse files
author
Judgment Release Bot
committed
Release: Merge staging to main
2 parents e8df0ec + bfa92b1 commit 8bb3d8e

File tree

9 files changed

+783
-4
lines changed

9 files changed

+783
-4
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ jobs:
3030
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
3131
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
3232
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
33+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
3334
JUDGMENT_DEV: true
3435

3536
steps:

‎.pre-commit-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
repos:
22
- repo: https://github.com/astral-sh/uv-pre-commit
3-
rev: 0.9.2
3+
rev: 0.9.7
44
hooks:
55
- id: uv-lock
66

77
- repo: https://github.com/astral-sh/ruff-pre-commit
8-
rev: v0.14.0
8+
rev: v0.14.3
99
hooks:
1010
- id: ruff
1111
name: ruff (linter)

‎src/judgeval/tracer/llm/llm_openai/chat_completions.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@
2525
immutable_wrap_sync_iterator,
2626
immutable_wrap_async_iterator,
2727
)
28-
from judgeval.tracer.llm.llm_openai.utils import openai_tokens_converter
28+
from judgeval.tracer.llm.llm_openai.utils import (
29+
openai_tokens_converter,
30+
set_cost_attribute,
31+
)
2932

3033
if TYPE_CHECKING:
3134
from judgeval.tracer import Tracer
@@ -90,6 +93,8 @@ def post_hook(ctx: Dict[str, Any], result: ChatCompletion) -> None:
9093
if prompt_tokens_details:
9194
cache_read = prompt_tokens_details.cached_tokens or 0
9295

96+
set_cost_attribute(span, usage_data)
97+
9398
prompt_tokens, completion_tokens, cache_read, cache_creation = (
9499
openai_tokens_converter(
95100
prompt_tokens,
@@ -195,6 +200,8 @@ def yield_hook(inner_ctx: Dict[str, Any], chunk: ChatCompletionChunk) -> None:
195200
if chunk.usage.prompt_tokens_details:
196201
cache_read = chunk.usage.prompt_tokens_details.cached_tokens or 0
197202

203+
set_cost_attribute(span, chunk.usage)
204+
198205
prompt_tokens, completion_tokens, cache_read, cache_creation = (
199206
openai_tokens_converter(
200207
prompt_tokens,
@@ -312,6 +319,8 @@ def post_hook(ctx: Dict[str, Any], result: ChatCompletion) -> None:
312319
if prompt_tokens_details:
313320
cache_read = prompt_tokens_details.cached_tokens or 0
314321

322+
set_cost_attribute(span, usage_data)
323+
315324
prompt_tokens, completion_tokens, cache_read, cache_creation = (
316325
openai_tokens_converter(
317326
prompt_tokens,
@@ -418,6 +427,8 @@ def yield_hook(inner_ctx: Dict[str, Any], chunk: ChatCompletionChunk) -> None:
418427
if chunk.usage.prompt_tokens_details:
419428
cache_read = chunk.usage.prompt_tokens_details.cached_tokens or 0
420429

430+
set_cost_attribute(span, chunk.usage)
431+
421432
prompt_tokens, completion_tokens, cache_read, cache_creation = (
422433
openai_tokens_converter(
423434
prompt_tokens,

‎src/judgeval/tracer/llm/llm_openai/responses.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@
2424
immutable_wrap_sync_iterator,
2525
immutable_wrap_async_iterator,
2626
)
27-
from judgeval.tracer.llm.llm_openai.utils import openai_tokens_converter
27+
from judgeval.tracer.llm.llm_openai.utils import (
28+
openai_tokens_converter,
29+
set_cost_attribute,
30+
)
2831

2932
if TYPE_CHECKING:
3033
from judgeval.tracer import Tracer
@@ -81,6 +84,7 @@ def post_hook(ctx: Dict[str, Any], result: Response) -> None:
8184
completion_tokens = usage_data.output_tokens or 0
8285
cache_read = usage_data.input_tokens_details.cached_tokens or 0
8386

87+
set_cost_attribute(span, usage_data)
8488
prompt_tokens, completion_tokens, cache_read, cache_creation = (
8589
openai_tokens_converter(
8690
prompt_tokens,
@@ -191,6 +195,7 @@ def yield_hook(inner_ctx: Dict[str, Any], chunk: Any) -> None:
191195
else 0
192196
)
193197

198+
set_cost_attribute(span, chunk.response.usage)
194199
prompt_tokens, completion_tokens, cache_read, cache_creation = (
195200
openai_tokens_converter(
196201
prompt_tokens,
@@ -312,6 +317,7 @@ def post_hook(ctx: Dict[str, Any], result: Response) -> None:
312317
completion_tokens = usage_data.output_tokens or 0
313318
cache_read = usage_data.input_tokens_details.cached_tokens or 0
314319

320+
set_cost_attribute(span, usage_data)
315321
prompt_tokens, completion_tokens, cache_read, cache_creation = (
316322
openai_tokens_converter(
317323
prompt_tokens,
@@ -424,6 +430,7 @@ def yield_hook(inner_ctx: Dict[str, Any], chunk: Any) -> None:
424430
else 0
425431
)
426432

433+
set_cost_attribute(span, chunk.response.usage)
427434
prompt_tokens, completion_tokens, cache_read, cache_creation = (
428435
openai_tokens_converter(
429436
prompt_tokens,

‎src/judgeval/tracer/llm/llm_openai/utils.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
from typing import Any
2+
from opentelemetry.trace import Span
3+
from judgeval.tracer.keys import AttributeKeys
4+
from judgeval.tracer.utils import set_span_attribute
5+
from judgeval.utils.serialize import safe_serialize
6+
7+
18
def openai_tokens_converter(
29
prompt_tokens: int,
310
completion_tokens: int,
@@ -20,3 +27,16 @@ def openai_tokens_converter(
2027
return prompt_tokens - cache_read, completion_tokens, cache_read, cache_creation
2128
else:
2229
return prompt_tokens, completion_tokens, cache_read, cache_creation
30+
31+
32+
def set_cost_attribute(span: Span, usage_data: Any) -> None:
33+
"""
34+
This is for OpenRouter case where the cost is provided in the usage data when they specify:
35+
extra_body={"usage": {"include": True}},
36+
"""
37+
if hasattr(usage_data, "cost") and usage_data.cost:
38+
set_span_attribute(
39+
span,
40+
AttributeKeys.JUDGMENT_USAGE_TOTAL_COST_USD,
41+
safe_serialize(usage_data.cost),
42+
)

‎src/tests/tracer/llm/llm_openrouter/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎src/tests/tracer/llm/llm_openrouter/conftest.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
"""OpenRouter-specific fixtures for tests."""
2+
3+
import pytest
4+
import os
5+
6+
pytest.importorskip("openai")
7+
8+
from openai import OpenAI, AsyncOpenAI
9+
from judgeval.tracer.llm.llm_openai.wrapper import (
10+
wrap_openai_client_sync,
11+
wrap_openai_client_async,
12+
)
13+
14+
15+
@pytest.fixture
16+
def openrouter_api_key():
17+
"""OpenRouter API key from environment"""
18+
api_key = os.getenv("OPENROUTER_API_KEY")
19+
if not api_key:
20+
pytest.skip("OPENROUTER_API_KEY environment variable not set")
21+
return api_key
22+
23+
24+
@pytest.fixture
25+
def sync_client(openrouter_api_key):
26+
"""Unwrapped sync OpenRouter client (using OpenAI SDK)"""
27+
return OpenAI(
28+
api_key=openrouter_api_key,
29+
base_url="https://openrouter.ai/api/v1",
30+
default_headers={
31+
"HTTP-Referer": os.getenv("OPENROUTER_APP_URL", "https://judgmentlabs.ai"),
32+
"X-Title": os.getenv("OPENROUTER_APP_NAME", "Judgeval Tests"),
33+
},
34+
)
35+
36+
37+
@pytest.fixture
38+
def async_client(openrouter_api_key):
39+
"""Unwrapped async OpenRouter client (using OpenAI SDK)"""
40+
return AsyncOpenAI(
41+
api_key=openrouter_api_key,
42+
base_url="https://openrouter.ai/api/v1",
43+
default_headers={
44+
"HTTP-Referer": os.getenv("OPENROUTER_APP_URL", "https://judgmentlabs.ai"),
45+
"X-Title": os.getenv("OPENROUTER_APP_NAME", "Judgeval Tests"),
46+
},
47+
)
48+
49+
50+
@pytest.fixture
51+
def wrapped_sync_client(tracer, sync_client):
52+
"""Wrapped sync OpenRouter client with tracer"""
53+
return wrap_openai_client_sync(tracer, sync_client)
54+
55+
56+
@pytest.fixture
57+
def wrapped_async_client(tracer, async_client):
58+
"""Wrapped async OpenRouter client with tracer"""
59+
return wrap_openai_client_async(tracer, async_client)
60+
61+
62+
@pytest.fixture(params=["wrapped", "unwrapped"], ids=["with_tracer", "without_tracer"])
63+
def sync_client_maybe_wrapped(request, tracer, sync_client):
64+
"""Parametrized fixture that yields both wrapped and unwrapped sync clients"""
65+
if request.param == "wrapped":
66+
return wrap_openai_client_sync(tracer, sync_client)
67+
return sync_client
68+
69+
70+
@pytest.fixture(params=["wrapped", "unwrapped"], ids=["with_tracer", "without_tracer"])
71+
def async_client_maybe_wrapped(request, tracer, async_client):
72+
"""Parametrized fixture that yields both wrapped and unwrapped async clients"""
73+
if request.param == "wrapped":
74+
return wrap_openai_client_async(tracer, async_client)
75+
return async_client

0 commit comments

Comments
 (0)