diff --git a/docs/api-reference/public-client.md b/docs/api-reference/public-client.md index 9a79d1e..31afc82 100644 --- a/docs/api-reference/public-client.md +++ b/docs/api-reference/public-client.md @@ -282,14 +282,12 @@ Returns an `Evaluation` object if found, `None` otherwise. See [Evaluations](eva ### `evaluations.get_many(...)` -Retrieves evaluations for a given organization and project with optional pagination, sorting, and filtering. +Retrieves evaluations with optional pagination, sorting, and filtering. #### Parameters | Parameter | Type | Required | Description | | ----------------- | -------------------------------- | -------- | ------------------------------------------------------------------ | -| `organization_id` | `str` | Yes | Organization ID (MongoDB ObjectID format) | -| `project_id` | `str` | Yes | Project ID (MongoDB ObjectID format) | | `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | | `page_size` | `int \| None` | No | Number of evaluations per page (default: 100, max: 500) | | `sort_by` | `str \| None` | No | Sort by field: `submittedAt`, `accuracy`, or `averageDuration` | @@ -325,10 +323,8 @@ if evaluation: for takeaway in evaluation.summary.analysis_summary.key_takeaways: print(f" - {takeaway}") -# List evaluations for an organization/project +# List successful evaluations sorted by accuracy response = client.evaluations.get_many( - organization_id="683e63925ef7e1c53c1f4b28", - project_id="683e63925ef7e1c53c1f4b29", status=EvaluationStatus.SUCCESS, sort_by="accuracy", order="desc", @@ -417,3 +413,47 @@ if comparison: print(f" Prompt: {result.prompt[:80]}...") print(f" Model 1 score: {result.score1}, Model 2 score: {result.score2}") ``` + +### `comparisons.compare_models(...)` + +Compares two models on a benchmark by automatically finding their most recent successful evaluations. This is a convenience method that wraps `compare()`. + +#### Parameters + +| Parameter | Type | Required | Description | +| ---------------- | ---------------------- | -------- | ------------------------------------------ | +| `benchmark_id` | `str` | Yes | Benchmark ID to compare on | +| `model_id_1` | `str` | Yes | First model ID | +| `model_id_2` | `str` | Yes | Second model ID | +| `page` | `int \| None` | No | Page number (1-based) | +| `page_size` | `int \| None` | No | Results per page | +| `outcome_filter` | `str \| None` | No | Filter by outcome (same options as `compare`) | +| `search` | `str \| None` | No | Search within results | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `ComparisonResponse` (same as `compare()`), or `None` if the comparison request fails. + +Raises `ValueError` if no successful evaluation is found for either model on the given benchmark. + +#### Example + +```python +from layerlens import PublicClient + +client = PublicClient() + +# Compare two models on AIME 2025 - no need to look up evaluation IDs +comparison = client.comparisons.compare_models( + benchmark_id="682bddc1e014f9fa440f8a91", + model_id_1="699f9761e014f9c3072b0513", + model_id_2="699f9761e014f9c3072b0512", + page=1, + page_size=10, +) + +if comparison: + print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct") + print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct") +``` diff --git a/examples/compare_evaluations.py b/examples/compare_evaluations.py index eb292b4..2293e8d 100644 --- a/examples/compare_evaluations.py +++ b/examples/compare_evaluations.py @@ -1,74 +1,49 @@ #!/usr/bin/env -S poetry run python -from layerlens import Stratix -from layerlens.models import EvaluationStatus +from layerlens import PublicClient def main(): - # Construct client (API key from env or inline) - client = Stratix() - - # --- Get successful evaluations to find a comparable pair - response = client.evaluations.get_many( - status=EvaluationStatus.SUCCESS, - sort_by="accuracy", - order="desc", - page_size=100, - ) - - if not response or len(response.evaluations) < 2: - print("Need at least 2 successful evaluations to compare, exiting") - return - - # Find two evaluations on the same benchmark - eval_1 = None - eval_2 = None - for i, e1 in enumerate(response.evaluations): - for e2 in response.evaluations[i + 1 :]: - if e1.benchmark_id == e2.benchmark_id and e1.id != e2.id: - eval_1 = e1 - eval_2 = e2 - break - if eval_1: - break - - if not eval_1 or not eval_2: - print("No two evaluations share the same benchmark, exiting") - return - - print(f"Comparing evaluations on the same benchmark ({eval_1.benchmark_id}):") - print(f" Evaluation 1: {eval_1.id} (accuracy={eval_1.accuracy:.2f}%)") - print(f" Evaluation 2: {eval_2.id} (accuracy={eval_2.accuracy:.2f}%)") - - # --- Get comparison results - comparison = client.public.comparisons.compare( - evaluation_id_1=eval_1.id, - evaluation_id_2=eval_2.id, + # Construct public client (API key from LAYERLENS_STRATIX_API_KEY env var or inline) + client = PublicClient() + + # --- Compare two models on a benchmark using compare_models + # Just provide the benchmark and two model IDs - the SDK automatically + # finds the most recent successful evaluation for each model. + benchmark_id = "682bddc1e014f9fa440f8a91" # AIME 2025 + model_id_1 = "699f9761e014f9c3072b0513" # Qwen3.5 27B + model_id_2 = "699f9761e014f9c3072b0512" # Qwen3.5 122B A10B + + print(f"Comparing models on benchmark {benchmark_id}...") + comparison = client.comparisons.compare_models( + benchmark_id=benchmark_id, + model_id_1=model_id_1, + model_id_2=model_id_2, page=1, page_size=10, ) if comparison: print(f"\n=== Comparison Summary ===") - print(f"Evaluation 1: {comparison.correct_count_1}/{comparison.total_results_1} correct") - print(f"Evaluation 2: {comparison.correct_count_2}/{comparison.total_results_2} correct") + print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct") + print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct") print(f"Total compared: {comparison.total_count}") - # --- Show individual results if comparison.results: print(f"\nFirst {len(comparison.results)} results:") for result in comparison.results: - score_indicator_1 = "✓" if result.score1 and result.score1 > 0.5 else "✗" - score_indicator_2 = "✓" if result.score2 and result.score2 > 0.5 else "✗" + s1 = "Y" if result.score1 and result.score1 > 0.5 else "N" + s2 = "Y" if result.score2 and result.score2 > 0.5 else "N" print(f" Prompt: {result.prompt[:80]}...") - print(f" Model 1: {score_indicator_1} (score={result.score1})") - print(f" Model 2: {score_indicator_2} (score={result.score2})") + print(f" Model 1: {s1} (score={result.score1})") + print(f" Model 2: {s2} (score={result.score2})") print() - # --- Filter by outcome: where only model 1 fails - comparison = client.public.comparisons.compare( - evaluation_id_1=eval_1.id, - evaluation_id_2=eval_2.id, + # --- Filter: where model 1 fails but model 2 succeeds + comparison = client.comparisons.compare_models( + benchmark_id=benchmark_id, + model_id_1=model_id_1, + model_id_2=model_id_2, outcome_filter="reference_fails", ) @@ -76,16 +51,18 @@ def main(): print(f"\n=== Where Model 1 Fails but Model 2 Succeeds ===") print(f"Found {comparison.total_count} such cases") - # --- Filter by outcome: where both models fail - comparison = client.public.comparisons.compare( - evaluation_id_1=eval_1.id, - evaluation_id_2=eval_2.id, - outcome_filter="both_fail", + # --- You can also compare using evaluation IDs directly + comparison = client.comparisons.compare( + evaluation_id_1="699f9938a03d70bf6607081f", # Qwen3.5 27B on AIME 2025 + evaluation_id_2="699f991ca782d00ebd666ba1", # Qwen3.5 122B A10B on AIME 2025 + page=1, + page_size=5, ) if comparison: - print(f"\n=== Where Both Models Fail ===") - print(f"Found {comparison.total_count} such cases") + print(f"\n=== Direct Comparison by Evaluation IDs ===") + print(f"Model 1: {comparison.correct_count_1}/{comparison.total_results_1} correct") + print(f"Model 2: {comparison.correct_count_2}/{comparison.total_results_2} correct") if __name__ == "__main__": diff --git a/examples/public_evaluations.py b/examples/public_evaluations.py index f28236f..a8eb588 100644 --- a/examples/public_evaluations.py +++ b/examples/public_evaluations.py @@ -30,13 +30,8 @@ def main(): else: print(f"Evaluation {evaluation_id} not found") - # --- List evaluations for a specific organization/project - organization_id = "683e63925ef7e1c53c1f4b28" - project_id = "683e63925ef7e1c53c1f4b29" - + # --- List latest evaluations response = client.evaluations.get_many( - organization_id=organization_id, - project_id=project_id, page=1, page_size=5, sort_by="submittedAt", @@ -49,8 +44,6 @@ def main(): # --- Filter by status (only successful) response = client.evaluations.get_many( - organization_id=organization_id, - project_id=project_id, status=EvaluationStatus.SUCCESS, sort_by="accuracy", order="desc", diff --git a/src/layerlens/resources/comparisons/comparisons.py b/src/layerlens/resources/comparisons/comparisons.py index eef469a..ed4851c 100644 --- a/src/layerlens/resources/comparisons/comparisons.py +++ b/src/layerlens/resources/comparisons/comparisons.py @@ -4,10 +4,19 @@ import httpx -from ...models import ComparisonResponse +from ...models import EvaluationStatus, ComparisonResponse, EvaluationsResponse from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource from ..._constants import DEFAULT_TIMEOUT +_OUTCOME_FILTER = Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"] + + +def _find_evaluation_id(response: Optional[EvaluationsResponse], model_id: str, benchmark_id: str) -> str: + """Extract the first evaluation ID from a response, or raise ValueError.""" + if not response or not response.evaluations: + raise ValueError(f"No successful evaluation found for model '{model_id}' on benchmark '{benchmark_id}'") + return str(response.evaluations[0].id) + class Comparisons(SyncPublicAPIResource): def compare( @@ -17,9 +26,7 @@ def compare( evaluation_id_2: str, page: Optional[int] = None, page_size: Optional[int] = None, - outcome_filter: Optional[ - Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"] - ] = None, + outcome_filter: Optional[_OUTCOME_FILTER] = None, search: Optional[str] = None, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[ComparisonResponse]: @@ -48,6 +55,58 @@ def compare( return ComparisonResponse.model_validate(resp) + def compare_models( + self, + *, + benchmark_id: str, + model_id_1: str, + model_id_2: str, + page: Optional[int] = None, + page_size: Optional[int] = None, + outcome_filter: Optional[_OUTCOME_FILTER] = None, + search: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[ComparisonResponse]: + """Compare two models on a benchmark by automatically finding their evaluations. + + Finds the most recent successful evaluation for each model on the given + benchmark, then compares the results side-by-side. + + Raises: + ValueError: If no successful evaluation is found for either model. + """ + resp1 = self._client.evaluations.get_many( + model_ids=[model_id_1], + benchmark_ids=[benchmark_id], + status=EvaluationStatus.SUCCESS, + sort_by="submittedAt", + order="desc", + page_size=1, + timeout=timeout, + ) + eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id) + + resp2 = self._client.evaluations.get_many( + model_ids=[model_id_2], + benchmark_ids=[benchmark_id], + status=EvaluationStatus.SUCCESS, + sort_by="submittedAt", + order="desc", + page_size=1, + timeout=timeout, + ) + eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id) + + return self.compare( + evaluation_id_1=eval_id_1, + evaluation_id_2=eval_id_2, + page=page, + page_size=page_size, + outcome_filter=outcome_filter, + search=search, + timeout=timeout, + ) + class AsyncComparisons(AsyncPublicAPIResource): async def compare( @@ -57,9 +116,7 @@ async def compare( evaluation_id_2: str, page: Optional[int] = None, page_size: Optional[int] = None, - outcome_filter: Optional[ - Literal["all", "both_succeed", "both_fail", "reference_fails", "comparison_fails"] - ] = None, + outcome_filter: Optional[_OUTCOME_FILTER] = None, search: Optional[str] = None, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[ComparisonResponse]: @@ -87,3 +144,55 @@ async def compare( return None return ComparisonResponse.model_validate(resp) + + async def compare_models( + self, + *, + benchmark_id: str, + model_id_1: str, + model_id_2: str, + page: Optional[int] = None, + page_size: Optional[int] = None, + outcome_filter: Optional[_OUTCOME_FILTER] = None, + search: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[ComparisonResponse]: + """Compare two models on a benchmark by automatically finding their evaluations. + + Finds the most recent successful evaluation for each model on the given + benchmark, then compares the results side-by-side. + + Raises: + ValueError: If no successful evaluation is found for either model. + """ + resp1 = await self._client.evaluations.get_many( + model_ids=[model_id_1], + benchmark_ids=[benchmark_id], + status=EvaluationStatus.SUCCESS, + sort_by="submittedAt", + order="desc", + page_size=1, + timeout=timeout, + ) + eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id) + + resp2 = await self._client.evaluations.get_many( + model_ids=[model_id_2], + benchmark_ids=[benchmark_id], + status=EvaluationStatus.SUCCESS, + sort_by="submittedAt", + order="desc", + page_size=1, + timeout=timeout, + ) + eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id) + + return await self.compare( + evaluation_id_1=eval_id_1, + evaluation_id_2=eval_id_2, + page=page, + page_size=page_size, + outcome_filter=outcome_filter, + search=search, + timeout=timeout, + ) diff --git a/src/layerlens/resources/public_evaluations/public_evaluations.py b/src/layerlens/resources/public_evaluations/public_evaluations.py index ddd1cdf..71f736d 100644 --- a/src/layerlens/resources/public_evaluations/public_evaluations.py +++ b/src/layerlens/resources/public_evaluations/public_evaluations.py @@ -37,8 +37,6 @@ def get_by_id( def get_many( self, *, - organization_id: str, - project_id: str, page: Optional[int] = None, page_size: Optional[int] = None, sort_by: Optional[Literal["submittedAt", "accuracy", "averageDuration"]] = None, @@ -52,8 +50,6 @@ def get_many( Get evaluations with optional pagination, sorting, and filtering. Args: - organization_id: Organization ID (required) - project_id: Project ID (required) page: Page number for pagination (1-based, defaults to 1 if not provided) page_size: Number of evaluations per page (default: 100, optional) sort_by: Sort evaluations by field (submittedAt, accuracy, averageDuration) @@ -66,10 +62,7 @@ def get_many( Returns: EvaluationsResponse object or None """ - params = { - "organizationID": organization_id, - "projectID": project_id, - } + params: dict[str, str] = {} effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE effective_page = page if page is not None else DEFAULT_PAGE @@ -137,8 +130,6 @@ async def get_by_id( async def get_many( self, *, - organization_id: str, - project_id: str, page: Optional[int] = None, page_size: Optional[int] = None, sort_by: Optional[Literal["submittedAt", "accuracy", "averageDuration"]] = None, @@ -152,8 +143,6 @@ async def get_many( Get evaluations with optional pagination, sorting, and filtering. Args: - organization_id: Organization ID (required) - project_id: Project ID (required) page: Page number for pagination (1-based, defaults to 1 if not provided) page_size: Number of evaluations per page (default: 100, optional) sort_by: Sort evaluations by field (submittedAt, accuracy, averageDuration) @@ -166,10 +155,7 @@ async def get_many( Returns: EvaluationsResponse object or None """ - params = { - "organizationID": organization_id, - "projectID": project_id, - } + params: dict[str, str] = {} effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE effective_page = page if page is not None else DEFAULT_PAGE diff --git a/tests/resources/test_comparisons.py b/tests/resources/test_comparisons.py new file mode 100644 index 0000000..85dfa2f --- /dev/null +++ b/tests/resources/test_comparisons.py @@ -0,0 +1,201 @@ +from unittest.mock import Mock + +import pytest + +from layerlens.models import ( + Evaluation, + Pagination, + EvaluationStatus, + ComparisonResponse, + EvaluationsResponse, +) +from layerlens.resources.comparisons.comparisons import Comparisons + + +def _make_eval(eval_id: str, model_id: str, benchmark_id: str) -> Evaluation: + return Evaluation( + id=eval_id, + status=EvaluationStatus.SUCCESS, + submitted_at=1640995200, + finished_at=1640995800, + model_id=model_id, + dataset_id=benchmark_id, + average_duration=2500, + accuracy=0.89, + ) + + +def _make_eval_response(evaluations: list[Evaluation]) -> EvaluationsResponse: + return EvaluationsResponse( + evaluations=evaluations, + pagination=Pagination( + page=1, + page_size=1, + total_pages=1, + total_count=len(evaluations), + ), + ) + + +class TestCompareModels: + """Test Comparisons.compare_models convenience method.""" + + @pytest.fixture + def mock_public_client(self): + client = Mock() + client.get_cast = Mock() + client.evaluations = Mock() + return client + + @pytest.fixture + def comparisons(self, mock_public_client): + return Comparisons(mock_public_client) + + def test_compare_models_success(self, comparisons, mock_public_client): + """compare_models finds evaluations for both models and calls compare.""" + eval1 = _make_eval("eval-1", "model-a", "bench-1") + eval2 = _make_eval("eval-2", "model-b", "bench-1") + + mock_public_client.evaluations.get_many.side_effect = [ + _make_eval_response([eval1]), + _make_eval_response([eval2]), + ] + + comparisons._get.return_value = { + "results": [], + "total_count": 0, + "correct_count_1": 5, + "total_results_1": 10, + "correct_count_2": 7, + "total_results_2": 10, + } + + result = comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + ) + + assert isinstance(result, ComparisonResponse) + + # Verify get_many was called correctly for both models + calls = mock_public_client.evaluations.get_many.call_args_list + assert len(calls) == 2 + + assert calls[0].kwargs["model_ids"] == ["model-a"] + assert calls[0].kwargs["benchmark_ids"] == ["bench-1"] + assert calls[0].kwargs["status"] == EvaluationStatus.SUCCESS + assert calls[0].kwargs["sort_by"] == "submittedAt" + assert calls[0].kwargs["order"] == "desc" + assert calls[0].kwargs["page_size"] == 1 + + assert calls[1].kwargs["model_ids"] == ["model-b"] + + # Verify compare was called with the found evaluation IDs + compare_call = comparisons._get.call_args + params = compare_call.kwargs.get("params") or compare_call[1].get("params") + assert params["evaluation_id_1"] == "eval-1" + assert params["evaluation_id_2"] == "eval-2" + + def test_compare_models_model_1_not_found(self, comparisons, mock_public_client): + """compare_models raises ValueError when model 1 has no evaluation.""" + mock_public_client.evaluations.get_many.return_value = _make_eval_response([]) + + with pytest.raises(ValueError, match="model-a"): + comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + ) + + def test_compare_models_model_2_not_found(self, comparisons, mock_public_client): + """compare_models raises ValueError when model 2 has no evaluation.""" + eval1 = _make_eval("eval-1", "model-a", "bench-1") + + mock_public_client.evaluations.get_many.side_effect = [ + _make_eval_response([eval1]), + _make_eval_response([]), + ] + + with pytest.raises(ValueError, match="model-b"): + comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + ) + + def test_compare_models_none_response(self, comparisons, mock_public_client): + """compare_models raises ValueError when get_many returns None.""" + mock_public_client.evaluations.get_many.return_value = None + + with pytest.raises(ValueError, match="model-a"): + comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + ) + + def test_compare_models_passes_through_params(self, comparisons, mock_public_client): + """compare_models forwards pagination, filter, and search to compare.""" + eval1 = _make_eval("eval-1", "model-a", "bench-1") + eval2 = _make_eval("eval-2", "model-b", "bench-1") + + mock_public_client.evaluations.get_many.side_effect = [ + _make_eval_response([eval1]), + _make_eval_response([eval2]), + ] + comparisons._get.return_value = { + "results": [], + "total_count": 0, + "correct_count_1": 0, + "total_results_1": 0, + "correct_count_2": 0, + "total_results_2": 0, + } + + comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + page=2, + page_size=50, + outcome_filter="both_succeed", + search="test query", + ) + + compare_call = comparisons._get.call_args + params = compare_call.kwargs.get("params") or compare_call[1].get("params") + assert params["page"] == "2" + assert params["pageSize"] == "50" + assert params["outcomeFilter"] == "both_succeed" + assert params["search"] == "test query" + + def test_compare_models_picks_most_recent(self, comparisons, mock_public_client): + """compare_models requests sort by submittedAt desc to get the most recent.""" + eval1 = _make_eval("eval-1", "model-a", "bench-1") + eval2 = _make_eval("eval-2", "model-b", "bench-1") + + mock_public_client.evaluations.get_many.side_effect = [ + _make_eval_response([eval1]), + _make_eval_response([eval2]), + ] + comparisons._get.return_value = { + "results": [], + "total_count": 0, + "correct_count_1": 0, + "total_results_1": 0, + "correct_count_2": 0, + "total_results_2": 0, + } + + comparisons.compare_models( + benchmark_id="bench-1", + model_id_1="model-a", + model_id_2="model-b", + ) + + for call in mock_public_client.evaluations.get_many.call_args_list: + assert call.kwargs["sort_by"] == "submittedAt" + assert call.kwargs["order"] == "desc" + assert call.kwargs["page_size"] == 1 + assert call.kwargs["status"] == EvaluationStatus.SUCCESS diff --git a/tests/resources/test_evaluations.py b/tests/resources/test_evaluations.py index 0d40f08..8a337eb 100644 --- a/tests/resources/test_evaluations.py +++ b/tests/resources/test_evaluations.py @@ -780,38 +780,18 @@ def test_get_many_success(self, public_evaluations, sample_evaluation_data): } public_evaluations._get.return_value = resp - result = public_evaluations.get_many( - organization_id="org-123", - project_id="proj-456", - ) + result = public_evaluations.get_many() assert isinstance(result, EvaluationsResponse) assert len(result.evaluations) == 1 assert result.evaluations[0].id == "eval-pub-123" - def test_get_many_sends_org_and_project(self, public_evaluations, sample_evaluation_data): - """get_many sends organizationID and projectID as params.""" - resp = {"evaluations": [sample_evaluation_data], "total_count": 1} - public_evaluations._get.return_value = resp - - public_evaluations.get_many( - organization_id="org-abc", - project_id="proj-xyz", - ) - - call_args = public_evaluations._get.call_args - params = call_args.kwargs.get("params") or call_args[1].get("params") - assert params["organizationID"] == "org-abc" - assert params["projectID"] == "proj-xyz" - def test_get_many_with_filters(self, public_evaluations, sample_evaluation_data): """get_many passes filter parameters correctly.""" resp = {"evaluations": [sample_evaluation_data], "total_count": 1} public_evaluations._get.return_value = resp public_evaluations.get_many( - organization_id="org-123", - project_id="proj-456", page=2, page_size=50, sort_by="accuracy", @@ -830,6 +810,8 @@ def test_get_many_with_filters(self, public_evaluations, sample_evaluation_data) assert params["models"] == "m1,m2" assert params["datasets"] == "b1" assert params["status"] == "success" + assert "organizationID" not in params + assert "projectID" not in params def test_get_many_pagination(self, public_evaluations, sample_evaluation_data): """get_many computes pagination correctly.""" @@ -837,8 +819,6 @@ def test_get_many_pagination(self, public_evaluations, sample_evaluation_data): public_evaluations._get.return_value = resp result = public_evaluations.get_many( - organization_id="org-123", - project_id="proj-456", page=1, page_size=10, ) @@ -852,10 +832,7 @@ def test_get_many_returns_none_on_invalid(self, public_evaluations): """get_many returns None when response is invalid.""" public_evaluations._get.return_value = "not-a-dict" - result = public_evaluations.get_many( - organization_id="org-123", - project_id="proj-456", - ) + result = public_evaluations.get_many() assert result is None @@ -864,10 +841,7 @@ def test_get_many_empty_results(self, public_evaluations): resp = {"evaluations": [], "total_count": 0} public_evaluations._get.return_value = resp - result = public_evaluations.get_many( - organization_id="org-123", - project_id="proj-456", - ) + result = public_evaluations.get_many() assert isinstance(result, EvaluationsResponse) assert len(result.evaluations) == 0