Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions .github/workflows/publish-sdk.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This workflow is used to publish the Python SDK to the actual PyPI.
# It is triggered by a tag push, and will only publish if the tag is valid.
# It is triggered by a tag push or manually, and will only publish if the tag is valid.
# The tag must match the format v*.*.*

name: Publish Python SDK
Expand All @@ -8,6 +8,12 @@ on:
push:
tags:
- "v*.*.*" # Trigger on version tags like v0.1.0 etc.
workflow_dispatch:
inputs:
release_tag:
description: "Release tag to publish (e.g. v1.2.0). Must match a tag that already exists."
required: true
type: string

jobs:
validate:
Expand All @@ -21,9 +27,13 @@ jobs:
fetch-depth: 0 # Fetch all history for checking branch
- name: Set release tag
id: set_release_tag
# ensure the tag is valid (matches code, is on main, etc)
# ensure the tag is valid (matches code, is on release branch, etc)
run: |
RELEASE_TAG=${GITHUB_REF#refs/tags/}
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RELEASE_TAG="${{ github.event.inputs.release_tag }}"
else
RELEASE_TAG=${GITHUB_REF#refs/tags/}
fi
echo "Using tag: $RELEASE_TAG"
./scripts/validate-release-tag.sh "$RELEASE_TAG"
echo "RELEASE_TAG=$RELEASE_TAG" >> $GITHUB_ENV
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/publish-to-aws.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ on:
# e.g. v1.0.0, v1.0.1, v1.0.0-rc1, etc.
- "v[0-9]+.[0-9]+.[0-9]+*"
workflow_dispatch:
inputs:
release_tag:
description: "Release tag to deploy (e.g. v1.2.0). Must match a tag that already exists."
required: true
type: string

env:
PACKAGE_NAME: layerlens
Expand All @@ -24,7 +29,11 @@ jobs:
id: set_release_tag
# ensure the tag is valid (matches code, is on release branch, etc)
run: |
RELEASE_TAG=${GITHUB_REF#refs/tags/}
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RELEASE_TAG="${{ github.event.inputs.release_tag }}"
else
RELEASE_TAG=${GITHUB_REF#refs/tags/}
fi
echo "Using tag: $RELEASE_TAG"
chmod +x ./scripts/validate_release_tag.sh
./scripts/validate_release_tag.sh "$RELEASE_TAG"
Expand Down
4 changes: 3 additions & 1 deletion examples/public_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ def main():
response = client.benchmarks.get(languages=["English"])
print(f"\nFound {response.total_count} English benchmarks")

# --- Filter by category
# --- Discover available filter values
if response.categories:
print(f"\nAvailable categories: {response.categories}")
if response.languages:
print(f"Available languages: {response.languages}")

# --- Search by name
response = client.benchmarks.get(query="mmlu")
Expand Down
9 changes: 9 additions & 0 deletions examples/public_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ def main():
response = client.models.get(include_deprecated=True)
print(f"\nTotal models (including deprecated): {response.total_count}")

# --- Discover available filter values
response = client.models.get(page=1, page_size=1)
print(f"\nAvailable filter values:")
print(f" Categories: {response.categories}")
print(f" Companies: {response.companies}")
print(f" Regions: {response.regions}")
print(f" Licenses: {response.licenses}")
print(f" Sizes: {response.sizes}")


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions scripts/push-release-tag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ done

git fetch --tags --prune

REPO_URL="https://github.com/LayerLens/atlas-python"
REPO_URL="https://github.com/LayerLens/stratix-python"
TAG_PREFIX="v"
COMMIT=$(git rev-parse --short HEAD)
VERSION=$(bash "$ROOT_DIR/scripts/get_version.sh")
Expand Down Expand Up @@ -68,5 +68,5 @@ git push origin "$TAG"

echo ""
echo "Tag ${TAG} has been created and pushed to origin. Check GitHub Actions for build progress:"
echo "https://github.com/LayerLens/atlas-python/actions/workflows/publish-sdk.yaml"
echo "https://github.com/LayerLens/stratix-python/actions/workflows/publish-sdk.yaml"
echo ""
2 changes: 1 addition & 1 deletion src/layerlens/_version.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = "1.2.0"
__version__ = "1.3.0"

# Will be templated during the build
__git_commit__ = "__GIT_COMMIT__"
14 changes: 7 additions & 7 deletions src/layerlens/models/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ class EvaluationDataset(BaseModel):
total_size: int = 0
training_size: int = 0
test_size: int = 0
characteristics: List[str] = []
characteristics: Optional[List[str]] = []


class EvaluationModelInfo(BaseModel):
Expand All @@ -45,24 +45,24 @@ class EvaluationModelInfo(BaseModel):


class PerformanceDetails(BaseModel):
strengths: List[str] = []
challenges: List[str] = []
strengths: Optional[List[str]] = []
challenges: Optional[List[str]] = []


class ErrorAnalysis(BaseModel):
common_failure_modes: List[str] = []
common_failure_modes: Optional[List[str]] = []
example: str = ""


class AnalysisSummary(BaseModel):
key_takeaways: List[str] = []
key_takeaways: Optional[List[str]] = []


class EvaluationSummary(BaseModel):
name: str = ""
goal: str = ""
metrics: List[EvaluationMetric] = []
task_types: List[EvaluationTaskType] = []
metrics: Optional[List[EvaluationMetric]] = []
task_types: Optional[List[EvaluationTaskType]] = []
dataset: Optional[EvaluationDataset] = None
model: Optional[EvaluationModelInfo] = None
performance_details: Optional[PerformanceDetails] = None
Expand Down
5 changes: 5 additions & 0 deletions src/layerlens/models/public.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ class PublicModelDetail(BaseModel):
class PublicModelsListResponse(BaseModel):
models: List[PublicModelDetail]
categories: List[str] = []
companies: List[str] = []
regions: List[str] = []
licenses: List[str] = []
sizes: List[str] = []
count: int
total_count: int

Expand All @@ -54,6 +58,7 @@ class PublicBenchmarkDetail(BaseModel):
class PublicBenchmarksListResponse(BaseModel):
datasets: List[PublicBenchmarkDetail]
categories: List[str] = []
languages: List[str] = []
count: int
total_count: int

Expand Down
24 changes: 16 additions & 8 deletions src/layerlens/resources/public_benchmarks/public_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource
from ..._constants import DEFAULT_TIMEOUT

DEFAULT_PAGE = 1
DEFAULT_PAGE_SIZE = 100
MAX_PAGE_SIZE = 500

DEFAULT_PROMPTS_PAGE_SIZE = 100
MAX_PROMPTS_PAGE_SIZE = 500

Expand Down Expand Up @@ -47,14 +51,16 @@ def get(
params["categories"] = ",".join(categories)
if languages:
params["languages"] = ",".join(languages)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down Expand Up @@ -170,14 +176,16 @@ async def get(
params["categories"] = ",".join(categories)
if languages:
params["languages"] = ",".join(languages)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down
24 changes: 16 additions & 8 deletions src/layerlens/resources/public_models/public_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
from ..._resource import SyncPublicAPIResource, AsyncPublicAPIResource
from ..._constants import DEFAULT_TIMEOUT

DEFAULT_PAGE = 1
DEFAULT_PAGE_SIZE = 100
MAX_PAGE_SIZE = 500


class PublicModelsResource(SyncPublicAPIResource):
def get(
Expand Down Expand Up @@ -50,14 +54,16 @@ def get(
params["licenses"] = ",".join(licenses)
if sizes:
params["sizes"] = ",".join(sizes)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down Expand Up @@ -115,14 +121,16 @@ async def get(
params["licenses"] = ",".join(licenses)
if sizes:
params["sizes"] = ",".join(sizes)
effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE
effective_page = page if page is not None else DEFAULT_PAGE

params["page"] = str(effective_page)
params["pageSize"] = str(effective_page_size)

if sort_by:
params["sortBy"] = sort_by
if order:
params["order"] = order
if page is not None:
params["page"] = str(page)
if page_size is not None:
params["pageSize"] = str(page_size)
if include_deprecated is not None:
params["include_deprecated"] = str(include_deprecated).lower()

Expand Down
Loading