Skip to content

Improve comparisons resource #51

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
m-peko merged 1 commit into from
Feb 27, 2026
Merged

Improve comparisons resource #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 46 additions & 6 deletions docs/api-reference/public-client.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,14 +282,12 @@ Returns an `Evaluation` object if found, `None` otherwise. See [Evaluations](eva

### `evaluations.get_many(...)`

Retrieves evaluations for a given organization and project with optional pagination, sorting, and filtering.
Retrieves evaluations with optional pagination, sorting, and filtering.

#### Parameters

| Parameter | Type | Required | Description |
| ----------------- | -------------------------------- | -------- | ------------------------------------------------------------------ |
| `organization_id` | `str` | Yes | Organization ID (MongoDB ObjectID format) |
| `project_id` | `str` | Yes | Project ID (MongoDB ObjectID format) |
| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) |
| `page_size` | `int \| None` | No | Number of evaluations per page (default: 100, max: 500) |
| `sort_by` | `str \| None` | No | Sort by field: `submittedAt`, `accuracy`, or `averageDuration` |
Expand Down Expand Up @@ -325,10 +323,8 @@ if evaluation:
for takeaway in evaluation.summary.analysis_summary.key_takeaways:
print(f" - {takeaway}")

# List evaluations for an organization/project
# List successful evaluations sorted by accuracy
response = client.evaluations.get_many(
organization_id="683e63925ef7e1c53c1f4b28",
project_id="683e63925ef7e1c53c1f4b29",
status=EvaluationStatus.SUCCESS,
sort_by="accuracy",
order="desc",
Expand Down Expand Up @@ -417,3 +413,47 @@ if comparison:
print(f" Prompt: {result.prompt[:80]}...")
print(f" Model 1 score: {result.score1}, Model 2 score: {result.score2}")
```

### `comparisons.compare_models(...)`

Compares two models on a benchmark by automatically finding their most recent successful evaluations. This is a convenience method that wraps `compare()`.

#### Parameters

| Parameter | Type | Required | Description |
| ---------------- | ---------------------- | -------- | ------------------------------------------ |
| `benchmark_id` | `str` | Yes | Benchmark ID to compare on |
| `model_id_1` | `str` | Yes | First model ID |
| `model_id_2` | `str` | Yes | Second model ID |
| `page` | `int \| None` | No | Page number (1-based) |
| `page_size` | `int \| None` | No | Results per page |
| `outcome_filter` | `str \| None` | No | Filter by outcome (same options as `compare`) |
| `search` | `str \| None` | No | Search within results |
| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout |

#### Returns

Returns a `ComparisonResponse` (same as `compare()`), or `None` if the comparison request fails.

Raises `ValueError` if no successful evaluation is found for either model on the given benchmark.

#### Example

```python
from layerlens import PublicClient

client = PublicClient()

# Compare two models on AIME 2025 - no need to look up evaluation IDs
comparison = client.comparisons.compare_models(
benchmark_id="682bddc1e014f9fa440f8a91",
model_id_1="699f9761e014f9c3072b0513",
model_id_2="699f9761e014f9c3072b0512",
page=1,
page_size=10,
)

if comparison:
print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")
```
95 changes: 36 additions & 59 deletions examples/compare_evaluations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,91 +1,68 @@
#!/usr/bin/env -S poetry run python

from layerlens import Stratix
from layerlens.models import EvaluationStatus
from layerlens import PublicClient


def main():
# Construct client (API key from env or inline)
client = Stratix()

# --- Get successful evaluations to find a comparable pair
response = client.evaluations.get_many(
status=EvaluationStatus.SUCCESS,
sort_by="accuracy",
order="desc",
page_size=100,
)

if not response or len(response.evaluations) < 2:
print("Need at least 2 successful evaluations to compare, exiting")
return

# Find two evaluations on the same benchmark
eval_1 = None
eval_2 = None
for i, e1 in enumerate(response.evaluations):
for e2 in response.evaluations[i + 1 :]:
if e1.benchmark_id == e2.benchmark_id and e1.id != e2.id:
eval_1 = e1
eval_2 = e2
break
if eval_1:
break

if not eval_1 or not eval_2:
print("No two evaluations share the same benchmark, exiting")
return

print(f"Comparing evaluations on the same benchmark ({eval_1.benchmark_id}):")
print(f" Evaluation 1: {eval_1.id} (accuracy={eval_1.accuracy:.2f}%)")
print(f" Evaluation 2: {eval_2.id} (accuracy={eval_2.accuracy:.2f}%)")

# --- Get comparison results
comparison = client.public.comparisons.compare(
evaluation_id_1=eval_1.id,
evaluation_id_2=eval_2.id,
# Construct public client (API key from LAYERLENS_STRATIX_API_KEY env var or inline)
client = PublicClient()

# --- Compare two models on a benchmark using compare_models
# Just provide the benchmark and two model IDs - the SDK automatically
# finds the most recent successful evaluation for each model.
benchmark_id = "682bddc1e014f9fa440f8a91" # AIME 2025
model_id_1 = "699f9761e014f9c3072b0513" # Qwen3.5 27B
model_id_2 = "699f9761e014f9c3072b0512" # Qwen3.5 122B A10B

print(f"Comparing models on benchmark {benchmark_id}...")
comparison = client.comparisons.compare_models(
benchmark_id=benchmark_id,
model_id_1=model_id_1,
model_id_2=model_id_2,
page=1,
page_size=10,
)

if comparison:
print(f"\n=== Comparison Summary ===")
print(f"Evaluation 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
print(f"Evaluation 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")
print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")
print(f"Total compared: {comparison.total_count}")

# --- Show individual results
if comparison.results:
print(f"\nFirst {len(comparison.results)} results:")
for result in comparison.results:
score_indicator_1 = "✓" if result.score1 and result.score1 > 0.5 else "✗"
score_indicator_2 = "✓" if result.score2 and result.score2 > 0.5 else "✗"
s1 = "Y" if result.score1 and result.score1 > 0.5 else "N"
s2 = "Y" if result.score2 and result.score2 > 0.5 else "N"
print(f" Prompt: {result.prompt[:80]}...")
print(f" Model 1: {score_indicator_1} (score={result.score1})")
print(f" Model 2: {score_indicator_2} (score={result.score2})")
print(f" Model 1: {s1} (score={result.score1})")
print(f" Model 2: {s2} (score={result.score2})")
print()

# --- Filter by outcome: where only model 1 fails
comparison = client.public.comparisons.compare(
evaluation_id_1=eval_1.id,
evaluation_id_2=eval_2.id,
# --- Filter: where model 1 fails but model 2 succeeds
comparison = client.comparisons.compare_models(
benchmark_id=benchmark_id,
model_id_1=model_id_1,
model_id_2=model_id_2,
outcome_filter="reference_fails",
)

if comparison:
print(f"\n=== Where Model 1 Fails but Model 2 Succeeds ===")
print(f"Found {comparison.total_count} such cases")

# --- Filter by outcome: where both models fail
comparison = client.public.comparisons.compare(
evaluation_id_1=eval_1.id,
evaluation_id_2=eval_2.id,
outcome_filter="both_fail",
# --- You can also compare using evaluation IDs directly
comparison = client.comparisons.compare(
evaluation_id_1="699f9938a03d70bf6607081f", # Qwen3.5 27B on AIME 2025
evaluation_id_2="699f991ca782d00ebd666ba1", # Qwen3.5 122B A10B on AIME 2025
page=1,
page_size=5,
)

if comparison:
print(f"\n=== Where Both Models Fail ===")
print(f"Found {comparison.total_count} such cases")
print(f"\n=== Direct Comparison by Evaluation IDs ===")
print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct")
print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct")


if __name__ == "__main__":
Expand Down
9 changes: 1 addition & 8 deletions examples/public_evaluations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,8 @@ def main():
else:
print(f"Evaluation {evaluation_id} not found")

# --- List evaluations for a specific organization/project
organization_id = "683e63925ef7e1c53c1f4b28"
project_id = "683e63925ef7e1c53c1f4b29"

# --- List latest evaluations
response = client.evaluations.get_many(
organization_id=organization_id,
project_id=project_id,
page=1,
page_size=5,
sort_by="submittedAt",
Expand All @@ -49,8 +44,6 @@ def main():

# --- Filter by status (only successful)
response = client.evaluations.get_many(
organization_id=organization_id,
project_id=project_id,
status=EvaluationStatus.SUCCESS,
sort_by="accuracy",
order="desc",
Expand Down
123 changes: 116 additions & 7 deletions src/layerlens/resources/comparisons/comparisons.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,19 @@

import httpx

from ...models import ComparisonResponse
from ...models import EvaluationStatus, ComparisonResponse, EvaluationsResponse
from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource
from ..._constants import DEFAULT_TIMEOUT

_OUTCOME_FILTER = Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"]


def _find_evaluation_id(response: Optional[EvaluationsResponse], model_id: str, benchmark_id: str) -> str:
"""Extract the first evaluation ID from a response, or raise ValueError."""
if not response or not response.evaluations:
raise ValueError(f"No successful evaluation found for model '{model_id}' on benchmark '{benchmark_id}'")
return str(response.evaluations[0].id)


class Comparisons(SyncPublicAPIResource):
def compare(
Expand All @@ -17,9 +26,7 @@ def compare(
evaluation_id_2: str,
page: Optional[int] = None,
page_size: Optional[int] = None,
outcome_filter: Optional[
Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"]
] = None,
outcome_filter: Optional[_OUTCOME_FILTER] = None,
search: Optional[str] = None,
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
) -> Optional[ComparisonResponse]:
Expand Down Expand Up @@ -48,6 +55,58 @@ def compare(

return ComparisonResponse.model_validate(resp)

def compare_models(
self,
*,
benchmark_id: str,
model_id_1: str,
model_id_2: str,
page: Optional[int] = None,
page_size: Optional[int] = None,
outcome_filter: Optional[_OUTCOME_FILTER] = None,
search: Optional[str] = None,
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
) -> Optional[ComparisonResponse]:
"""Compare two models on a benchmark by automatically finding their evaluations.

Finds the most recent successful evaluation for each model on the given
benchmark, then compares the results side-by-side.

Raises:
ValueError: If no successful evaluation is found for either model.
"""
resp1 = self._client.evaluations.get_many(
model_ids=[model_id_1],
benchmark_ids=[benchmark_id],
status=EvaluationStatus.SUCCESS,
sort_by="submittedAt",
order="desc",
page_size=1,
timeout=timeout,
)
eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id)

resp2 = self._client.evaluations.get_many(
model_ids=[model_id_2],
benchmark_ids=[benchmark_id],
status=EvaluationStatus.SUCCESS,
sort_by="submittedAt",
order="desc",
page_size=1,
timeout=timeout,
)
eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id)

return self.compare(
evaluation_id_1=eval_id_1,
evaluation_id_2=eval_id_2,
page=page,
page_size=page_size,
outcome_filter=outcome_filter,
search=search,
timeout=timeout,
)


class AsyncComparisons(AsyncPublicAPIResource):
async def compare(
Expand All @@ -57,9 +116,7 @@ async def compare(
evaluation_id_2: str,
page: Optional[int] = None,
page_size: Optional[int] = None,
outcome_filter: Optional[
Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"]
] = None,
outcome_filter: Optional[_OUTCOME_FILTER] = None,
search: Optional[str] = None,
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
) -> Optional[ComparisonResponse]:
Expand Down Expand Up @@ -87,3 +144,55 @@ async def compare(
return None

return ComparisonResponse.model_validate(resp)

async def compare_models(
self,
*,
benchmark_id: str,
model_id_1: str,
model_id_2: str,
page: Optional[int] = None,
page_size: Optional[int] = None,
outcome_filter: Optional[_OUTCOME_FILTER] = None,
search: Optional[str] = None,
timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT,
) -> Optional[ComparisonResponse]:
"""Compare two models on a benchmark by automatically finding their evaluations.

Finds the most recent successful evaluation for each model on the given
benchmark, then compares the results side-by-side.

Raises:
ValueError: If no successful evaluation is found for either model.
"""
resp1 = await self._client.evaluations.get_many(
model_ids=[model_id_1],
benchmark_ids=[benchmark_id],
status=EvaluationStatus.SUCCESS,
sort_by="submittedAt",
order="desc",
page_size=1,
timeout=timeout,
)
eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id)

resp2 = await self._client.evaluations.get_many(
model_ids=[model_id_2],
benchmark_ids=[benchmark_id],
status=EvaluationStatus.SUCCESS,
sort_by="submittedAt",
order="desc",
page_size=1,
timeout=timeout,
)
eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id)

return await self.compare(
evaluation_id_1=eval_id_1,
evaluation_id_2=eval_id_2,
page=page,
page_size=page_size,
outcome_filter=outcome_filter,
search=search,
timeout=timeout,
)
Loading
Loading