Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion examples/public_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ def main():
response = client.benchmarks.get(languages=["English"])
print(f"\nFound {response.total_count} English benchmarks")

# --- Filter by category
# --- Discover available filter values
if response.categories:
print(f"\nAvailable categories: {response.categories}")
if response.languages:
print(f"Available languages: {response.languages}")

# --- Search by name
response = client.benchmarks.get(query="mmlu")
Expand Down
9 changes: 9 additions & 0 deletions examples/public_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ def main():
response = client.models.get(include_deprecated=True)
print(f"\nTotal models (including deprecated): {response.total_count}")

# --- Discover available filter values
response = client.models.get(page=1, page_size=1)
print(f"\nAvailable filter values:")
print(f" Categories: {response.categories}")
print(f" Companies: {response.companies}")
print(f" Regions: {response.regions}")
print(f" Licenses: {response.licenses}")
print(f" Sizes: {response.sizes}")


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions scripts/push-release-tag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ done

git fetch --tags --prune

REPO_URL="https://github.com/LayerLens/atlas-python"
REPO_URL="https://github.com/LayerLens/stratix-python"
TAG_PREFIX="v"
COMMIT=$(git rev-parse --short HEAD)
VERSION=$(bash "$ROOT_DIR/scripts/get_version.sh")
Expand Down Expand Up @@ -68,5 +68,5 @@ git push origin "$TAG"

echo ""
echo "Tag ${TAG} has been created and pushed to origin. Check GitHub Actions for build progress:"
echo "https://github.com/LayerLens/atlas-python/actions/workflows/publish-sdk.yaml"
echo "https://github.com/LayerLens/stratix-python/actions/workflows/publish-sdk.yaml"
echo ""
14 changes: 7 additions & 7 deletions src/layerlens/models/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class EvaluationDataset(BaseModel):
total_size: int = 0
training_size: int = 0
test_size: int = 0
characteristics: List[str] = []
characteristics: Optional[List[str]] = []


class EvaluationModelInfo(BaseModel):
Expand All @@ -45,24 +45,24 @@ class EvaluationModelInfo(BaseModel):


class PerformanceDetails(BaseModel):
strengths: List[str] = []
challenges: List[str] = []
strengths: Optional[List[str]] = []
challenges: Optional[List[str]] = []


class ErrorAnalysis(BaseModel):
common_failure_modes: List[str] = []
common_failure_modes: Optional[List[str]] = []
example: str = ""


class AnalysisSummary(BaseModel):
key_takeaways: List[str] = []
key_takeaways: Optional[List[str]] = []


class EvaluationSummary(BaseModel):
name: str = ""
goal: str = ""
metrics: List[EvaluationMetric] = []
task_types: List[EvaluationTaskType] = []
metrics: Optional[List[EvaluationMetric]] = []
task_types: Optional[List[EvaluationTaskType]] = []
dataset: Optional[EvaluationDataset] = None
model: Optional[EvaluationModelInfo] = None
performance_details: Optional[PerformanceDetails] = None
Expand Down
5 changes: 5 additions & 0 deletions src/layerlens/models/public.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ class PublicModelDetail(BaseModel):
class PublicModelsListResponse(BaseModel):
models: List[PublicModelDetail]
categories: List[str] = []
companies: List[str] = []
regions: List[str] = []
licenses: List[str] = []
sizes: List[str] = []
count: int
total_count: int

Expand All @@ -54,6 +58,7 @@ class PublicBenchmarkDetail(BaseModel):
class PublicBenchmarksListResponse(BaseModel):
datasets: List[PublicBenchmarkDetail]
categories: List[str] = []
languages: List[str] = []
count: int
total_count: int

Expand Down
24 changes: 16 additions & 8 deletions src/layerlens/resources/public_benchmarks/public_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource
from ..._constants import DEFAULT_TIMEOUT

DEFAULT_PAGE = 1
DEFAULT_PAGE_SIZE = 100
MAX_PAGE_SIZE = 500

DEFAULT_PROMPTS_PAGE_SIZE = 100
MAX_PROMPTS_PAGE_SIZE = 500

Expand Down Expand Up @@ -47,14 +51,16 @@ def get(
params["categories"] = ",".join(categories)
if languages:
params["languages"] = ",".join(languages)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down Expand Up @@ -170,14 +176,16 @@ async def get(
params["categories"] = ",".join(categories)
if languages:
params["languages"] = ",".join(languages)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down
24 changes: 16 additions & 8 deletions src/layerlens/resources/public_models/public_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource
from ..._constants import DEFAULT_TIMEOUT

DEFAULT_PAGE = 1
DEFAULT_PAGE_SIZE = 100
MAX_PAGE_SIZE = 500


class PublicModelsResource(SyncPublicAPIResource):
def get(
Expand Down Expand Up @@ -50,14 +54,16 @@ def get(
params["licenses"] = ",".join(licenses)
if sizes:
params["sizes"] = ",".join(sizes)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down Expand Up @@ -115,14 +121,16 @@ async def get(
params["licenses"] = ",".join(licenses)
if sizes:
params["sizes"] = ",".join(sizes)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down
Loading