Skip to content

Commit 9a773ff

Browse files
alanzhang25alanzhang25
authored
dont throw (#614)
1 parent a4dc6d7 commit 9a773ff

File tree

11 files changed

+165
-81
lines changed

11 files changed

+165
-81
lines changed

‎src/e2etests/conftest.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ def project_name():
3939
def client(project_name: str) -> JudgmentClient:
4040
"""Create a single JudgmentClient instance for all tests."""
4141
# Setup
42+
if not API_KEY or not ORGANIZATION_ID:
43+
pytest.skip(
44+
"JUDGMENT_API_KEY or JUDGMENT_ORG_ID not set", allow_module_level=True
45+
)
4246
client = JudgmentClient(api_key=API_KEY, organization_id=ORGANIZATION_ID)
4347
create_project(project_name=project_name)
4448
yield client

‎src/judgeval/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ def upload_custom_scorer(
146146
requirements_text = f.read()
147147

148148
try:
149+
if not self.api_key or not self.organization_id:
150+
raise ValueError("Judgment API key and organization ID are required")
149151
client = JudgmentSyncClient(
150152
api_key=self.api_key,
151153
organization_id=self.organization_id,

‎src/judgeval/api/api_types.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# generated by datamodel-codegen:
22
# filename: .openapi.json
3-
# timestamp: 2025-10-23T10:50:55+00:00
3+
# timestamp: 2025-10-25T22:30:20+00:00
44

55
from __future__ import annotations
66
from typing import Any, Dict, List, Literal, Optional, TypedDict, Union
@@ -24,16 +24,6 @@ class DatasetsFetch(TypedDict):
2424
project_name: str
2525

2626

27-
class DatasetsTableRow(TypedDict):
28-
dataset_id: str
29-
name: str
30-
created_at: str
31-
kind: Literal["trace", "example"]
32-
entries: int
33-
creator: str
34-
bucket_name: NotRequired[Optional[str]]
35-
36-
3727
class ProjectAdd(TypedDict):
3828
project_name: str
3929

@@ -302,6 +292,15 @@ class DatasetInsertExamples(TypedDict):
302292
project_name: str
303293

304294

295+
class DatasetInfo(TypedDict):
296+
dataset_id: str
297+
name: str
298+
created_at: str
299+
kind: DatasetKind
300+
entries: int
301+
creator: str
302+
303+
305304
class DatasetCreate(TypedDict):
306305
name: str
307306
dataset_kind: DatasetKind
@@ -348,6 +347,7 @@ class OtelTraceListItem(TypedDict):
348347
llm_cost: NotRequired[Optional[float]]
349348
error: NotRequired[str]
350349
scores: NotRequired[List[OtelSpanListItemScores]]
350+
rules_invoked: NotRequired[List[str]]
351351
customer_id: NotRequired[Optional[str]]
352352
input: NotRequired[Optional[str]]
353353
output: NotRequired[Optional[str]]

‎src/judgeval/data/judgment_types.py‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# generated by datamodel-codegen:
22
# filename: .openapi.json
3-
# timestamp: 2025-10-23T10:50:54+00:00
3+
# timestamp: 2025-10-25T22:30:19+00:00
44

55
from __future__ import annotations
66
from typing import Annotated, Any, Dict, List, Optional, Union
@@ -26,21 +26,6 @@ class DatasetsFetch(BaseModel):
2626
project_name: Annotated[str, Field(title="Project Name")]
2727

2828

29-
class Kind(Enum):
30-
trace = "trace"
31-
example = "example"
32-
33-
34-
class DatasetsTableRow(BaseModel):
35-
dataset_id: Annotated[str, Field(title="Dataset Id")]
36-
name: Annotated[str, Field(title="Name")]
37-
created_at: Annotated[str, Field(title="Created At")]
38-
kind: Annotated[Kind, Field(title="Kind")]
39-
entries: Annotated[int, Field(title="Entries")]
40-
creator: Annotated[str, Field(title="Creator")]
41-
bucket_name: Annotated[Optional[str], Field(title="Bucket Name")] = None
42-
43-
4429
class ProjectAdd(BaseModel):
4530
project_name: Annotated[str, Field(title="Project Name")]
4631

@@ -336,6 +321,15 @@ class DatasetInsertExamples(BaseModel):
336321
project_name: Annotated[str, Field(title="Project Name")]
337322

338323

324+
class DatasetInfo(BaseModel):
325+
dataset_id: Annotated[str, Field(title="Dataset Id")]
326+
name: Annotated[str, Field(title="Name")]
327+
created_at: Annotated[str, Field(title="Created At")]
328+
kind: DatasetKind
329+
entries: Annotated[int, Field(title="Entries")]
330+
creator: Annotated[str, Field(title="Creator")]
331+
332+
339333
class DatasetCreate(BaseModel):
340334
name: Annotated[str, Field(title="Name")]
341335
dataset_kind: DatasetKind
@@ -386,6 +380,7 @@ class OtelTraceListItem(BaseModel):
386380
scores: Annotated[
387381
Optional[List[OtelSpanListItemScores]], Field(title="Scores")
388382
] = []
383+
rules_invoked: Annotated[Optional[List[str]], Field(title="Rules Invoked")] = []
389384
customer_id: Annotated[Optional[str], Field(title="Customer Id")] = None
390385
input: Annotated[Optional[str], Field(title="Input")] = None
391386
output: Annotated[Optional[str], Field(title="Output")] = None

‎src/judgeval/dataset/__init__.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,17 @@ class Dataset:
3232
dataset_kind: DatasetKind = DatasetKind.example
3333
examples: Optional[List[Example]] = None
3434
traces: Optional[List[Trace]] = None
35-
judgment_api_key: str = JUDGMENT_API_KEY or ""
36-
organization_id: str = JUDGMENT_ORG_ID or ""
35+
judgment_api_key: str | None = JUDGMENT_API_KEY
36+
organization_id: str | None = JUDGMENT_ORG_ID
3737

3838
@classmethod
3939
def get(
4040
cls,
4141
name: str,
4242
project_name: str,
4343
):
44+
if not cls.judgment_api_key or not cls.organization_id:
45+
raise ValueError("Judgment API key and organization ID are required")
4446
client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
4547
dataset = client.datasets_pull_for_judgeval(
4648
{
@@ -102,6 +104,8 @@ def create(
102104
examples: List[Example] = [],
103105
overwrite: bool = False,
104106
):
107+
if not cls.judgment_api_key or not cls.organization_id:
108+
raise ValueError("Judgment API key and organization ID are required")
105109
if not examples:
106110
examples = []
107111

@@ -125,6 +129,8 @@ def create(
125129

126130
@classmethod
127131
def list(cls, project_name: str):
132+
if not cls.judgment_api_key or not cls.organization_id:
133+
raise ValueError("Judgment API key and organization ID are required")
128134
client = JudgmentSyncClient(cls.judgment_api_key, cls.organization_id)
129135
datasets = client.datasets_pull_all_for_judgeval({"project_name": project_name})
130136

@@ -173,6 +179,9 @@ def add_examples(self, examples: List[Example]) -> None:
173179
if not isinstance(examples, list):
174180
raise TypeError("examples must be a list")
175181

182+
if not self.judgment_api_key or not self.organization_id:
183+
raise ValueError("Judgment API key and organization ID are required")
184+
176185
client = JudgmentSyncClient(self.judgment_api_key, self.organization_id)
177186
client.datasets_insert_examples_for_judgeval(
178187
{

‎src/judgeval/env.py‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,8 @@ def optional_env_var(var_name: str, default: str | None = None) -> str | None:
1919
return os.getenv(var_name, default)
2020

2121

22-
def required_env_var(var_name: str) -> str:
23-
value = os.getenv(var_name)
24-
if value is None:
25-
raise EnvironmentError(
26-
f"Environment variable '{var_name}' is required but not set."
27-
)
28-
return value
29-
30-
31-
JUDGMENT_API_KEY = required_env_var("JUDGMENT_API_KEY")
32-
JUDGMENT_ORG_ID = required_env_var("JUDGMENT_ORG_ID")
22+
JUDGMENT_API_KEY = optional_env_var("JUDGMENT_API_KEY")
23+
JUDGMENT_ORG_ID = optional_env_var("JUDGMENT_ORG_ID")
3324
JUDGMENT_API_URL = optional_env_var("JUDGMENT_API_URL", "https://api.judgmentlabs.ai")
3425

3526
JUDGMENT_DEFAULT_GPT_MODEL = optional_env_var("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-5")

‎src/judgeval/evaluation/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ def _poll_evaluation_until_complete(
112112

113113
poll_count = 0
114114
exception_count = 0
115+
if not JUDGMENT_API_KEY or not JUDGMENT_ORG_ID:
116+
raise ValueError("Judgment API key and organization ID are required")
115117
api_client = JudgmentSyncClient(JUDGMENT_API_KEY, JUDGMENT_ORG_ID)
116118
while poll_count < max_poll_count:
117119
poll_count += 1
@@ -222,6 +224,8 @@ def run_eval(
222224
)
223225
t.start()
224226
try:
227+
if not JUDGMENT_API_KEY or not JUDGMENT_ORG_ID:
228+
raise ValueError("Judgment API key and organization ID are required")
225229
api_client = JudgmentSyncClient(JUDGMENT_API_KEY, JUDGMENT_ORG_ID)
226230
response = api_client.add_to_run_eval_queue_examples(
227231
evaluation_run.model_dump(warnings=False) # type: ignore

‎src/judgeval/prompt/__init__.py‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ def push_prompt(
1919
name: str,
2020
prompt: str,
2121
tags: List[str],
22-
judgment_api_key: str = JUDGMENT_API_KEY,
23-
organization_id: str = JUDGMENT_ORG_ID,
22+
judgment_api_key: str | None = JUDGMENT_API_KEY,
23+
organization_id: str | None = JUDGMENT_ORG_ID,
2424
) -> tuple[str, Optional[str], str]:
25+
if not judgment_api_key or not organization_id:
26+
raise ValueError("Judgment API key and organization ID are required")
2527
client = JudgmentSyncClient(judgment_api_key, organization_id)
2628
try:
2729
project_id = _resolve_project_id(
@@ -55,9 +57,11 @@ def fetch_prompt(
5557
name: str,
5658
commit_id: Optional[str] = None,
5759
tag: Optional[str] = None,
58-
judgment_api_key: str = JUDGMENT_API_KEY,
59-
organization_id: str = JUDGMENT_ORG_ID,
60+
judgment_api_key: str | None = JUDGMENT_API_KEY,
61+
organization_id: str | None = JUDGMENT_ORG_ID,
6062
) -> Optional[PromptCommitInfo]:
63+
if not judgment_api_key or not organization_id:
64+
raise ValueError("Judgment API key and organization ID are required")
6165
client = JudgmentSyncClient(judgment_api_key, organization_id)
6266
try:
6367
project_id = _resolve_project_id(
@@ -89,9 +93,11 @@ def tag_prompt(
8993
name: str,
9094
commit_id: str,
9195
tags: List[str],
92-
judgment_api_key: str = JUDGMENT_API_KEY,
93-
organization_id: str = JUDGMENT_ORG_ID,
96+
judgment_api_key: str | None = JUDGMENT_API_KEY,
97+
organization_id: str | None = JUDGMENT_ORG_ID,
9498
) -> PromptTagResponse:
99+
if not judgment_api_key or not organization_id:
100+
raise ValueError("Judgment API key and organization ID are required")
95101
client = JudgmentSyncClient(judgment_api_key, organization_id)
96102
try:
97103
project_id = _resolve_project_id(
@@ -124,9 +130,11 @@ def untag_prompt(
124130
project_name: str,
125131
name: str,
126132
tags: List[str],
127-
judgment_api_key: str = JUDGMENT_API_KEY,
128-
organization_id: str = JUDGMENT_ORG_ID,
133+
judgment_api_key: str | None = JUDGMENT_API_KEY,
134+
organization_id: str | None = JUDGMENT_ORG_ID,
129135
) -> PromptUntagResponse:
136+
if not judgment_api_key or not organization_id:
137+
raise ValueError("Judgment API key and organization ID are required")
130138
client = JudgmentSyncClient(judgment_api_key, organization_id)
131139
try:
132140
project_id = _resolve_project_id(
@@ -153,9 +161,11 @@ def untag_prompt(
153161
def list_prompt(
154162
project_name: str,
155163
name: str,
156-
judgment_api_key: str = JUDGMENT_API_KEY,
157-
organization_id: str = JUDGMENT_ORG_ID,
164+
judgment_api_key: str | None = JUDGMENT_API_KEY,
165+
organization_id: str | None = JUDGMENT_ORG_ID,
158166
) -> PromptVersionsResponse:
167+
if not judgment_api_key or not organization_id:
168+
raise ValueError("Judgment API key and organization ID are required")
159169
client = JudgmentSyncClient(judgment_api_key, organization_id)
160170
try:
161171
project_id = _resolve_project_id(
@@ -201,8 +211,8 @@ def create(
201211
name: str,
202212
prompt: str,
203213
tags: Optional[List[str]] = None,
204-
judgment_api_key: str = JUDGMENT_API_KEY,
205-
organization_id: str = JUDGMENT_ORG_ID,
214+
judgment_api_key: str | None = JUDGMENT_API_KEY,
215+
organization_id: str | None = JUDGMENT_ORG_ID,
206216
):
207217
if tags is None:
208218
tags = []
@@ -225,8 +235,8 @@ def get(
225235
name: str,
226236
commit_id: Optional[str] = None,
227237
tag: Optional[str] = None,
228-
judgment_api_key: str = JUDGMENT_API_KEY,
229-
organization_id: str = JUDGMENT_ORG_ID,
238+
judgment_api_key: str | None = JUDGMENT_API_KEY,
239+
organization_id: str | None = JUDGMENT_ORG_ID,
230240
):
231241
if commit_id is not None and tag is not None:
232242
raise ValueError(
@@ -262,8 +272,8 @@ def tag(
262272
name: str,
263273
commit_id: str,
264274
tags: List[str],
265-
judgment_api_key: str = JUDGMENT_API_KEY,
266-
organization_id: str = JUDGMENT_ORG_ID,
275+
judgment_api_key: str | None = JUDGMENT_API_KEY,
276+
organization_id: str | None = JUDGMENT_ORG_ID,
267277
):
268278
prompt_config = tag_prompt(
269279
project_name, name, commit_id, tags, judgment_api_key, organization_id
@@ -276,8 +286,8 @@ def untag(
276286
project_name: str,
277287
name: str,
278288
tags: List[str],
279-
judgment_api_key: str = JUDGMENT_API_KEY,
280-
organization_id: str = JUDGMENT_ORG_ID,
289+
judgment_api_key: str | None = JUDGMENT_API_KEY,
290+
organization_id: str | None = JUDGMENT_ORG_ID,
281291
):
282292
prompt_config = untag_prompt(
283293
project_name, name, tags, judgment_api_key, organization_id
@@ -289,8 +299,8 @@ def list(
289299
cls,
290300
project_name: str,
291301
name: str,
292-
judgment_api_key: str = JUDGMENT_API_KEY,
293-
organization_id: str = JUDGMENT_ORG_ID,
302+
judgment_api_key: str | None = JUDGMENT_API_KEY,
303+
organization_id: str | None = JUDGMENT_ORG_ID,
294304
):
295305
prompt_configs = list_prompt(
296306
project_name, name, judgment_api_key, organization_id

0 commit comments

Comments
 (0)