diff --git a/src/layerlens/_version.py b/src/layerlens/_version.py index 1f6dff1..ddcf27c 100644 --- a/src/layerlens/_version.py +++ b/src/layerlens/_version.py @@ -1,4 +1,4 @@ -__version__ = "1.3.3" +__version__ = "1.4.0" # Will be templated during the build __git_commit__ = "__GIT_COMMIT__" diff --git a/src/layerlens/resources/comparisons/comparisons.py b/src/layerlens/resources/comparisons/comparisons.py index c05a752..ce62077 100644 --- a/src/layerlens/resources/comparisons/comparisons.py +++ b/src/layerlens/resources/comparisons/comparisons.py @@ -82,6 +82,7 @@ def compare_models( sort_by="submitted_at", order="desc", page_size=1, + unique=True, timeout=timeout, ) eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id) @@ -93,6 +94,7 @@ def compare_models( sort_by="submitted_at", order="desc", page_size=1, + unique=True, timeout=timeout, ) eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id) @@ -172,6 +174,7 @@ async def compare_models( sort_by="submitted_at", order="desc", page_size=1, + unique=True, timeout=timeout, ) eval_id_1 = _find_evaluation_id(resp1, model_id_1, benchmark_id) @@ -183,6 +186,7 @@ async def compare_models( sort_by="submitted_at", order="desc", page_size=1, + unique=True, timeout=timeout, ) eval_id_2 = _find_evaluation_id(resp2, model_id_2, benchmark_id) diff --git a/src/layerlens/resources/evaluations/evaluations.py b/src/layerlens/resources/evaluations/evaluations.py index 0fb1e34..4a405b6 100644 --- a/src/layerlens/resources/evaluations/evaluations.py +++ b/src/layerlens/resources/evaluations/evaluations.py @@ -86,6 +86,7 @@ def get_many( model_ids: Optional[List[str]] = None, benchmark_ids: Optional[List[str]] = None, status: Optional[EvaluationStatus] = None, + unique: bool = False, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[EvaluationsResponse]: """ @@ -99,6 +100,7 @@ def get_many( model_ids: Filter by model IDs benchmark_ids: Filter by benchmark/dataset IDs status: Filter by evaluation status + unique: If True, deduplicate by model+dataset keeping only the latest evaluation per pair timeout: Request timeout Returns: @@ -125,6 +127,8 @@ def get_many( params["datasets"] = ",".join(benchmark_ids) if status: params["status"] = status.value + if unique: + params["unique"] = "true" resp = self._get( f"/evaluations", @@ -241,6 +245,7 @@ async def get_many( model_ids: Optional[List[str]] = None, benchmark_ids: Optional[List[str]] = None, status: Optional[EvaluationStatus] = None, + unique: bool = False, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[EvaluationsResponse]: """ @@ -254,6 +259,7 @@ async def get_many( model_ids: Filter by model IDs benchmark_ids: Filter by benchmark/dataset IDs status: Filter by evaluation status + unique: If True, deduplicate by model+dataset keeping only the latest evaluation per pair timeout: Request timeout Returns: @@ -280,6 +286,8 @@ async def get_many( params["datasets"] = ",".join(benchmark_ids) if status: params["status"] = status.value + if unique: + params["unique"] = "true" resp = await self._get( f"/evaluations", diff --git a/src/layerlens/resources/public_evaluations/public_evaluations.py b/src/layerlens/resources/public_evaluations/public_evaluations.py index 86880ab..d218ab9 100644 --- a/src/layerlens/resources/public_evaluations/public_evaluations.py +++ b/src/layerlens/resources/public_evaluations/public_evaluations.py @@ -44,6 +44,7 @@ def get_many( model_ids: Optional[List[str]] = None, benchmark_ids: Optional[List[str]] = None, status: Optional[EvaluationStatus] = None, + unique: bool = False, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[EvaluationsResponse]: """ @@ -57,6 +58,7 @@ def get_many( model_ids: Filter by model IDs benchmark_ids: Filter by benchmark/dataset IDs status: Filter by evaluation status + unique: If True, deduplicate by model+dataset keeping only the latest evaluation per pair timeout: Request timeout Returns: @@ -80,6 +82,8 @@ def get_many( params["datasets"] = ",".join(benchmark_ids) if status: params["status"] = status.value + if unique: + params["unique"] = "true" resp = self._get( "/evaluations", @@ -137,6 +141,7 @@ async def get_many( model_ids: Optional[List[str]] = None, benchmark_ids: Optional[List[str]] = None, status: Optional[EvaluationStatus] = None, + unique: bool = False, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> Optional[EvaluationsResponse]: """ @@ -150,6 +155,7 @@ async def get_many( model_ids: Filter by model IDs benchmark_ids: Filter by benchmark/dataset IDs status: Filter by evaluation status + unique: If True, deduplicate by model+dataset keeping only the latest evaluation per pair timeout: Request timeout Returns: @@ -173,6 +179,8 @@ async def get_many( params["datasets"] = ",".join(benchmark_ids) if status: params["status"] = status.value + if unique: + params["unique"] = "true" resp = await self._get( "/evaluations", diff --git a/tests/resources/test_comparisons.py b/tests/resources/test_comparisons.py index 0773b35..b955294 100644 --- a/tests/resources/test_comparisons.py +++ b/tests/resources/test_comparisons.py @@ -88,6 +88,7 @@ def test_compare_models_success(self, comparisons, mock_public_client): assert calls[0].kwargs["sort_by"] == "submitted_at" assert calls[0].kwargs["order"] == "desc" assert calls[0].kwargs["page_size"] == 1 + assert calls[0].kwargs["unique"] is True assert calls[1].kwargs["model_ids"] == ["model-b"] @@ -199,3 +200,4 @@ def test_compare_models_picks_most_recent(self, comparisons, mock_public_client) assert call.kwargs["order"] == "desc" assert call.kwargs["page_size"] == 1 assert call.kwargs["status"] == EvaluationStatus.SUCCESS + assert call.kwargs["unique"] is True