From 5152c86a44ba47a85f0cc7a62d29339fd1461300 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 14:54:34 -0400 Subject: [PATCH 01/15] updated getting started docs with clear steps and functional examples --- SUMMARY.md | 3 - docs/README.md | 130 ++++++++++++++++++++++----------- docs/examples/README.md | 2 +- docs/getting-started/README.md | 2 +- docs/security/README.md | 2 +- docs/troubleshooting/README.md | 2 +- examples/async_client.py | 38 ++++------ examples/client_simple.py | 36 ++++----- 8 files changed, 121 insertions(+), 94 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index baeac3d..55db615 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,7 +1,4 @@ # Table of contents - -* [Atlas Python API library](README.md) -* [CONTRIBUTING](CONTRIBUTING.md) * [Atlas Python SDK Documentation](docs/README.md) * [Table of Contents - Atlas Python SDK](docs/SUMMARY.md) * [api-reference](docs/api-reference/README.md) diff --git a/docs/README.md b/docs/README.md index a57f613..d97f071 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,61 +1,105 @@ -# Atlas Python SDK Documentation +# Layerlens Python SDK Documentation -Welcome to the official documentation for the Atlas Python SDK. This library provides convenient access to the LayerLens Atlas REST API from any Python 3.8+ application. +Welcome to the official documentation for the Layerlens Python SDK for the atlas platform. This library provides convenient programmatic to the Atlas platform from any Python 3.8+ application. ## What is Atlas? -Atlas is LayerLens's evaluation platform that allows you to benchmark AI models against various datasets and metrics. The Python SDK provides a synchronous HTTP client powered by [httpx](https://github.com/encode/httpx) and [Pydantic](https://pydantic.dev/) models for type-safe API interactions. +Atlas is an evaluation platform that allows you to benchmark AI models against various datasets and metrics. The Python SDK provides two HTTP clients (syncronous and asynchronous) powered by [httpx](https://github.com/encode/httpx) and [Pydantic](https://pydantic.dev/) models for type-safe API interactions. -## Key Features +## Quick Start + +### Install LayerLens python sdk +Install the layerlens python sdk using the following command +```bash +pip install layerlens --index-url https://sdk.layerlens.ai +``` -- **Simple Authentication**: Secure API key-based authentication -- **Type Safety**: Full Pydantic model support for all API responses -- **Comprehensive Error Handling**: Detailed exception hierarchy for different error scenarios -- **Configurable Timeouts**: Fine-grained timeout control for different operations -- **Environment Variable Support**: Easy configuration through environment variables -- **Python 3.8+ Compatibility**: Works with modern Python versions +### Generate an api key on the atlas platform -## Quick Start +Login to your organization at [app.layerlens.ai](app.layerlens.ai) to generate an api key. Admin users of organizations can generate a keys in the settings page. + +Run this command to add your API key to your environment: + +```bash +export LAYERLENS_ATLAS_API_KEY="YOUR_API_KEY" +``` + +### Running an evaluation on the atlas platform + +Before triggering an evaluation using the sdk ensure that the model and benchmark you are trying to evaluate has been added to your organizations dashboard on the atlas platform. + +#### Using synchronous client ```python -import os from atlas import Atlas -# Initialize the client -client = Atlas( - api_key=os.environ.get("LAYERLENS_ATLAS_API_KEY"), -) - -# Create an evaluation -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" -) - -# Get results -if evaluation: - results_data = client.results.get(evaluation_id=evaluation.id) - if results_data: - print(f"Evaluation completed with {len(results_data.results)} results") - print(f"Total results available: {results_data.pagination.total_count}") - if results_data.pagination.total_pages > 1: - print(f"Results span {results_data.pagination.total_pages} pages - use pagination to access all") + # Construct sync client + client = Atlas() + + # --- Models replace with the model name you want to run + models = client.models.get(type="public", name="gpt-4o") + + if not models: + print("gpt-4o not found on organization, exiting") + + model = models[0] + + # --- Benchmarks replace with the benchmark name you want to run + benchmarks = client.benchmarks.get(type="public", name="simpleQA") + + if not benchmarks: + print("SimpleQA benchmark not found on organization, exiting") + + benchmark = benchmarks[0] + + # --- Create evaluation + evaluation = client.evaluations.create( + model=model, + benchmark=benchmark, + ) ``` -## Navigation -- **[Getting Started](getting-started/)** - Installation, setup, and your first API call -- **[API Reference](api-reference/)** - Complete documentation of all available methods -- **[Code Examples](examples/)** - Practical examples for common use cases -- **[Troubleshooting](troubleshooting/)** - Solutions to common issues -- **[Security](security/)** - Best practices for secure API usage +#### Using Async Client + +```python +import asyncio +from atlas import AsyncAtlas + +async def run_evaluation_async(): + # Construct async client + client = AsyncAtlas() + + # --- Models replace with the model name you want to run + models = await client.models.get(type="public",name="gpt-4o") + print(f"Models found: {models}") -## Support + if not models: + print("gpt-4o not found, exiting") + return -- **LayerLens Support**: Contact support through your LayerLens dashboard -- **Documentation**: Visit [docs.layerlens.com](https://docs.layerlens.com) for additional resources -- **API Status**: Check the [LayerLens status page](https://status.layerlens.com) for service updates + model = models[0] + # --- Benchmarks replace with the benchmark name you want to run + benchmarks = await client.benchmarks.get(type="public", name="simpleQA") -## License + if not benchmarks: + print("SimpleQA benchmark not found, exiting") + return -This SDK is released under the MIT License. + benchmark = benchmarks[0] + + # --- Create evaluation + evaluation = await client.evaluations.create( + model=model, + benchmark=benchmark, + ) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Next steps + +- **[API Reference](api-reference/)** - Complete documentation of all available methods +- **[Code Examples](examples/)** - Practical examples for common use cases +- **[Troubleshooting](troubleshooting/)** - Solutions to common issues diff --git a/docs/examples/README.md b/docs/examples/README.md index 2e21a6f..65afe60 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -1,2 +1,2 @@ -# examples +# Examples diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index c375f56..c453564 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -1,2 +1,2 @@ -# getting-started +# Getting-started diff --git a/docs/security/README.md b/docs/security/README.md index d9e4c5c..3ebfc6f 100644 --- a/docs/security/README.md +++ b/docs/security/README.md @@ -1,2 +1,2 @@ -# security +# Security diff --git a/docs/troubleshooting/README.md b/docs/troubleshooting/README.md index 9d6859b..8554af6 100644 --- a/docs/troubleshooting/README.md +++ b/docs/troubleshooting/README.md @@ -1,2 +1,2 @@ -# troubleshooting +# Troubleshooting diff --git a/examples/async_client.py b/examples/async_client.py index ccc2efb..579084a 100644 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -10,34 +10,28 @@ async def main(): client = AsyncAtlas() # --- Models - models = await client.models.get() - print(f"Found {len(models)} models") + models = await client.models.get(type="public",name="gpt-4o") + print(f"Models found: {models}") + if not models: + print("gpt-4o not found, exiting") + return + + model = models[0] # --- Benchmarks - benchmarks = await client.benchmarks.get() - print(f"Found {len(benchmarks)} benchmarks") + benchmarks = await client.benchmarks.get(type="public", name="simpleQA") + + if not benchmarks: + print("SimpleQA benchmark not found, exiting") + return + + benchmark = benchmarks[0] # --- Create evaluation evaluation = await client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], - ) - print(f"Created evaluation {evaluation.id}, status={evaluation.status}") - - # --- Wait for completion - evaluation = await client.evaluations.wait_for_completion( - evaluation, - interval_seconds=10, - timeout_seconds=600, # 10 minutes + model=model, + benchmark=benchmark, ) - print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") - - # --- Results - if evaluation.is_success: - results = await client.results.get(evaluation=evaluation) - print("Results:", results) - else: - print("Evaluation did not succeed, no results to show.") if __name__ == "__main__": diff --git a/examples/client_simple.py b/examples/client_simple.py index 12134ad..7560658 100644 --- a/examples/client_simple.py +++ b/examples/client_simple.py @@ -6,31 +6,23 @@ client = Atlas() # --- Models -models = client.models.get() -print(f"Found {len(models)} models") +models = client.models.get(type="public", name="gpt-4o") + +if not models: + print("gpt-4o not found, exiting") + +model = models[0] # --- Benchmarks -benchmarks = client.benchmarks.get() -print(f"Found {len(benchmarks)} benchmarks") +benchmarks = client.benchmarks.get(type="public", name="simpleQA") -# --- Create evaluation -evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], -) +if not benchmarks: + print("SimpleQA benchmark not found, exiting") -print(f"Created evaluation {evaluation.id}, status={evaluation.status}") +benchmark = benchmarks[0] -# --- Wait for completion -evaluation.wait_for_completion( - interval_seconds=10, - timeout_seconds=600, # 10 minutes +# --- Create evaluation +evaluation = client.evaluations.create( + model=model, + benchmark=benchmark, ) -print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") - -# --- Results -if evaluation.is_success: - results = evaluation.get_results() - print("Results:", results) -else: - print("Evaluation did not succeed, no results to show.") From 9f7b5a61b4484f4a7ddf0fd33041e9d261552d49 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 15:15:18 -0400 Subject: [PATCH 02/15] move readme --- README.md => src/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README.md => src/README.md (100%) diff --git a/README.md b/src/README.md similarity index 100% rename from README.md rename to src/README.md From c3c2e0f2e9918e7a09eabead4fb395084b407a42 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 15:16:56 -0400 Subject: [PATCH 03/15] fix broken links --- docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index d97f071..82a083c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,7 +16,7 @@ pip install layerlens --index-url https://sdk.layerlens.ai ### Generate an api key on the atlas platform -Login to your organization at [app.layerlens.ai](app.layerlens.ai) to generate an api key. Admin users of organizations can generate a keys in the settings page. +Login to your organization at [app.layerlens.ai](https://app.layerlens.ai) to generate an api key. Admin users of organizations can generate a keys in the settings page. Run this command to add your API key to your environment: @@ -26,7 +26,7 @@ export LAYERLENS_ATLAS_API_KEY="YOUR_API_KEY" ### Running an evaluation on the atlas platform -Before triggering an evaluation using the sdk ensure that the model and benchmark you are trying to evaluate has been added to your organizations dashboard on the atlas platform. +Before triggering an evaluation using the sdk, login to your organization at [app.layerlens.ai](https://app.layerlens.ai) to ensure that the model and benchmark you are trying to evaluate has been added to your organizations dashboard. #### Using synchronous client From d852ce736fc1c4e3483dce2ad626a6d223829868 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 15:17:44 -0400 Subject: [PATCH 04/15] update summary section --- SUMMARY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SUMMARY.md b/SUMMARY.md index 55db615..34b0e65 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,5 +1,5 @@ # Table of contents -* [Atlas Python SDK Documentation](docs/README.md) +* [Layerlens Python SDK Documentation](docs/README.md) * [Table of Contents - Atlas Python SDK](docs/SUMMARY.md) * [api-reference](docs/api-reference/README.md) * [Client Configuration](docs/api-reference/client.md) From cbdab6eefdc61005d963143140ead1dc2e953e74 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 15:19:34 -0400 Subject: [PATCH 05/15] remove unneeded scetions --- SUMMARY.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index 34b0e65..6e9eaac 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,26 +1,16 @@ # Table of contents * [Layerlens Python SDK Documentation](docs/README.md) - * [Table of Contents - Atlas Python SDK](docs/SUMMARY.md) - * [api-reference](docs/api-reference/README.md) + * [Api-reference](docs/api-reference/README.md) * [Client Configuration](docs/api-reference/client.md) * [Error Handling](docs/api-reference/errors.md) * [Evaluations](docs/api-reference/evaluations.md) * [Models & Benchmarks](docs/api-reference/models-benchmarks.md) * [Results](docs/api-reference/results.md) - * [examples](docs/examples/README.md) - * [Advanced Usage](docs/examples/advanced-usage.md) + * [Examples](docs/examples/README.md) * [Creating Evaluations](docs/examples/creating-evaluations.md) * [Retrieving Results](docs/examples/retrieving-results.md) * [Working with Timeouts](docs/examples/timeouts.md) - * [getting-started](docs/getting-started/README.md) - * [Authentication & Configuration](docs/getting-started/authentication.md) - * [Installation](docs/getting-started/installation.md) - * [Quick Start Guide](docs/getting-started/quickstart.md) - * [security](docs/security/README.md) - * [API Key Management](docs/security/api-key-management.md) - * [Environment Variables](docs/security/environment-variables.md) - * [Rate Limiting](docs/security/rate-limiting.md) - * [troubleshooting](docs/troubleshooting/README.md) + * [Troubleshooting](docs/troubleshooting/README.md) * [Authentication Problems](docs/troubleshooting/authentication.md) * [Common Issues](docs/troubleshooting/common-issues.md) * [Error Codes Reference](docs/troubleshooting/error-codes.md) From ce89034c858b6f95bffb8da6e74e819a730578e0 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 15:41:21 -0400 Subject: [PATCH 06/15] update client docs --- docs/api-reference/client.md | 252 +++++------------------------------ 1 file changed, 35 insertions(+), 217 deletions(-) diff --git a/docs/api-reference/client.md b/docs/api-reference/client.md index 155b659..100512d 100644 --- a/docs/api-reference/client.md +++ b/docs/api-reference/client.md @@ -1,22 +1,39 @@ # Client Configuration -The `Atlas` class is the main entry point for interacting with the LayerLens Atlas API. This page covers client initialization, configuration options, and advanced usage patterns. +The `Atlas` (syncronous) and `AsyncAtlas` (asyncronous) classes are the main entry points for interacting with the LayerLens Atlas sdk. This page covers client initialization, configuration options, and usage patterns. ## Basic Usage +### Syncronous Client + ```python from atlas import Atlas -# Using environment variables (recommended) +# Construct syncronous client +# Loads for api key from the "LAYERLENS_ATLAS_API_KEY" enviornment variable client = Atlas() # Explicit configuration client = Atlas(api_key="your_api_key") ``` +### Asyncronous Client + +```python +import asyncio +from atlas import AsyncAtlas + +# Construct async client +# Loads for api key from the "LAYERLENS_ATLAS_API_KEY" enviornment variable +client = AsyncAtlas() + +# Explicit configuration +client = AsyncAtlas(api_key="your_api_key") +``` + ## Constructor Parameters -### `Atlas(api_key, base_url, timeout)` +### `Atlas(api_key, base_url, timeout)` and `AsyncAtlas(api_key, base_url, timeout)` | Parameter | Type | Required | Default | Description | | ---------- | -------------------------------- | -------- | ------------- | ----------------------------- | @@ -32,7 +49,6 @@ The client automatically loads configuration from these environment variables: ```bash LAYERLENS_ATLAS_API_KEY="your_api_key_here" -LAYERLENS_ATLAS_BASE_URL="https://custom-endpoint.com/api/v1" # Optional ``` ## Timeout Configuration @@ -46,228 +62,30 @@ from atlas import Atlas client = Atlas(timeout=30.0) ``` -### Advanced Timeout Configuration - -```python -import httpx -from atlas import Atlas - -client = Atlas( - timeout=httpx.Timeout( - connect=5.0, # Connection timeout: 5 seconds - read=60.0, # Read timeout: 60 seconds - write=30.0, # Write timeout: 30 seconds - pool=10.0 # Connection pool timeout: 10 seconds - ) -) -``` - ### Per-Request Timeout Override ```python client = Atlas() -# Override timeout for a specific request -evaluation = client.with_options(timeout=120.0).evaluations.create( - model="gpt-4", - benchmark="mmlu" -) -``` - -## Client Methods - -### `copy(**kwargs)` - -Create a new client instance with modified configuration: - -```python -# Base client -client = Atlas(api_key="key1") - -# Create a copy with different timeout -slow_client = client.copy(timeout=300.0) # 5 minutes -``` - -### `with_options(**kwargs)` - -Temporarily override client options for a single request chain: - -```python -client = Atlas() - -# Use different timeout for this request only -evaluation = client.with_options(timeout=60.0).evaluations.create( - model="gpt-4", - benchmark="mmlu" -) - -# Back to original timeout for subsequent requests -results = client.results.get(evaluation_id=evaluation.id) -``` - -## Resource Access - -The client provides access to different API resources through properties: - -```python -client = Atlas() - -# Access evaluations resource -client.evaluations.create(model="gpt-4", benchmark="mmlu") - -# Access results resource -client.results.get(evaluation_id="eval_123") -``` - -Available resources: - -- `client.evaluations` - Create and manage evaluations -- `client.results` - Retrieve evaluation results -- More resources coming soon... - -## Error Handling - -The client raises specific exceptions for different error conditions: - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - evaluation = client.evaluations.create(model="invalid", benchmark="invalid") -except atlas.AuthenticationError: - # 401 - Invalid API key - print("Authentication failed") -except atlas.PermissionDeniedError: - # 403 - Valid API key, insufficient permissions - print("Permission denied") -except atlas.NotFoundError: - # 404 - Resource not found - print("Model or benchmark not found") -except atlas.RateLimitError: - # 429 - Too many requests - print("Rate limit exceeded") -except atlas.InternalServerError: - # 500+ - Server error - print("Server error occurred") -except atlas.APIConnectionError: - # Network/connection issues - print("Connection failed") -except atlas.APITimeoutError: - # Request timeout - print("Request timed out") -``` +# --- Models replace with the model name you want to run +models = client.models.get(type="public", name="gpt-4o") -## Authentication Headers +if not models: + print("gpt-4o not found on organization, exiting") -The client automatically handles authentication by adding the required headers: - -```python -# The client adds this header to all requests: -# x-api-key: your_api_key_value -``` +model = models[0] -You don't need to manually handle authentication headers. +# --- Benchmarks replace with the benchmark name you want to run +benchmarks = client.benchmarks.get(type="public", name="simpleQA") -## Base URL Configuration +if not benchmarks: + print("SimpleQA benchmark not found on organization, exiting") -### Default Base URL - -The client uses the default LayerLens Atlas API endpoint unless overridden. - -### Custom Base URL - -For enterprise or self-hosted deployments: - -```python -from atlas import Atlas +benchmark = benchmarks[0] -client = Atlas( - base_url="https://your-atlas-instance.com/api/v1" +# Override timeout for a specific request +evaluation = client.with_options(timeout=120.0).evaluations.create( + model=model, + benchmark=benchmark ) - -# Or via environment variable -# LAYERLENS_ATLAS_BASE_URL="https://your-atlas-instance.com/api/v1" -client = Atlas() # Will use custom base URL from environment -``` - -## Best Practices - -### 1. Use Environment Variables - -```python -# ✅ Good - secure and flexible -client = Atlas() - -# ❌ Bad - hardcoded credentials -client = Atlas(api_key="hardcoded_key") -``` - -### 2. Configure Appropriate Timeouts - -```python -# ✅ Good - reasonable timeout for evaluation creation -client = Atlas(timeout=120.0) # 2 minutes - -# ❌ Bad - too short for long-running operations -client = Atlas(timeout=5.0) # 5 seconds might be too short -``` - -### 3. Handle Errors Gracefully - -```python -# ✅ Good - specific error handling -try: - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.RateLimitError: - time.sleep(60) # Wait before retrying - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.APIError as e: - logger.error(f"API error: {e}") - raise -``` - -### 4. Reuse Client Instances - -```python -# ✅ Good - reuse the same client -client = Atlas() -eval1 = client.evaluations.create(model="gpt-4", benchmark="mmlu") -eval2 = client.evaluations.create(model="claude-3", benchmark="hellaswag") - -# ❌ Bad - creating new clients unnecessarily -client1 = Atlas() -eval1 = client1.evaluations.create(model="gpt-4", benchmark="mmlu") -client2 = Atlas() # Unnecessary -eval2 = client2.evaluations.create(model="claude-3", benchmark="hellaswag") -``` - -## Thread Safety - -The Atlas client is thread-safe and can be shared across multiple threads: - -```python -import threading -from atlas import Atlas - -client = Atlas() - -def create_evaluation(model_name): - evaluation = client.evaluations.create( - model=model_name, - benchmark="mmlu" - ) - print(f"Created evaluation for {model_name}: {evaluation.id}") - -# Safe to use the same client across threads -threads = [] -for model in ["gpt-4", "claude-3", "llama-2"]: - thread = threading.Thread(target=create_evaluation, args=(model,)) - threads.append(thread) - thread.start() - -for thread in threads: - thread.join() -``` +``` \ No newline at end of file From 8984cad7a91b5596313daacaf9b9d8ac44869003 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 16:04:09 -0400 Subject: [PATCH 07/15] simplify model/benchmark page --- docs/api-reference/models-benchmarks.md | 287 +++--------------------- 1 file changed, 29 insertions(+), 258 deletions(-) diff --git a/docs/api-reference/models-benchmarks.md b/docs/api-reference/models-benchmarks.md index 7baacfa..855dab1 100644 --- a/docs/api-reference/models-benchmarks.md +++ b/docs/api-reference/models-benchmarks.md @@ -1,30 +1,43 @@ # Models & Benchmarks -This page provides reference information about available models and benchmarks in the Atlas platform, along with guidance on selecting appropriate combinations for your evaluations. +This page provides reference information about accessing the available models and benchmarks on your organization using the layerlens python sdk. ## Overview -Atlas evaluations require two key components: -- **Model**: The AI model you want to evaluate -- **Benchmark**: The dataset/test suite to evaluate the model against +Running evaluations on the atlas platform require a model and benchmark to be selected. The availability of models and benchmarks depends on your organizations configuration. ### Finding Available Models and Benchmarks -#### Check the Atlas Dashboard +#### 1. Using the Atlas Dashboard The most reliable way to find available models and benchmarks: -1. Log into your Atlas dashboard -2. Navigate to the evaluation creation page -3. View dropdown lists of available models and benchmarks +1. Log into your Atlas dashboard. +2. Navigate to the evaluation creation page. +3. View dropdown lists of available models and benchmarks. + +#### 2. Using the python sdk + +```python +from atlas import Atlas + +# Construct sync client (API key from env or inline) +client = Atlas() + +# --- Models +models = client.models.get() + +# --- Benchmarks +benchmarks = client.benchmarks.get() +``` ## Models ### `get(type=None, name=None, companies=None, regions=None, licenses=None, timeout=None)` -Retrieves a list of available models with optional filtering parameters. +Retrieves a list of available models with optional filtering parameters. Both the `Atlas` and `AsyncAtlas` clients have this method. #### Parameters @@ -39,55 +52,8 @@ Retrieves a list of available models with optional filtering parameters. #### Returns -Returns a `List[Model]` containing available models that match the filter criteria. Returns `None` if no models are found or if there's an error. - -#### Examples +Returns an `Optional[List[Model]]` - a list of `Model` objects that match the filter criteria. Returns an empty list `[]` if no models match the criteria, or `None` if there's an error. -##### Get All Models - -```python -from atlas import Atlas - -client = Atlas() - -# Get all available models (both custom and public) -models = client.models.get() - -if models: - print(f"Found {len(models)} models:") - for model in models: - print(f" {model.id} - {model.name} ({model.company})") -else: - print("No models available") -``` - -##### Filter by Company - -```python -# Get models from specific companies -openai_models = client.models.get(companies=["OpenAI"]) -anthropic_models = client.models.get(companies=["Anthropic"]) - -# Get models from multiple companies -major_models = client.models.get(companies=["OpenAI", "Anthropic", "Google"]) - -if major_models: - for model in major_models: - print(f"{model.name} by {model.company}") -``` - -##### Search by Name - -```python -# Search for models with "gpt" in the name -gpt_models = client.models.get(name=["gpt"]) - - -if gpt_models: - print("GPT models found:") - for model in gpt_models: - print(f" {model.id} - {model.name}") -``` #### Model Object Properties @@ -97,70 +63,17 @@ Each `Model` object in the returned list contains: |----------|------|-------------| | `id` | `str` | Unique model identifier for use in evaluations | | `name` | `str` | Human-readable model name | -| `company` | `str` | Company/provider that created the model | -| `type` | `str` | Model type ("custom" or "public") | -| `region` | `str` | Supported region | -| `license` | `str` | License type | - -#### Error Handling - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - models = client.models.get() - if models: - print(f"Retrieved {len(models)} models") - else: - print("No models available or error occurred") -except atlas.AuthenticationError: - print("Authentication failed - check your API key") -except atlas.PermissionDeniedError: - print("No permission to access models") -except atlas.APIConnectionError as e: - print(f"Connection error: {e}") -except atlas.APIError as e: - print(f"API error: {e}") -``` - -### Model Identification - -Models are identified by string IDs that you pass to the `evaluations.create()` method: +| `key` | `str` | Unique model key/identifier that is similar to the name | +| `description` | `str` | Text description of the model | -```python -from atlas import Atlas - -client = Atlas() - -# Using model ID -evaluation = client.evaluations.create( - model="gpt-4", # Model ID - benchmark="mmlu" -) -``` - -### Model Information -When you create an evaluation, the response includes detailed model information: -```python -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") - -if evaluation: - print(f"Model ID: {evaluation.model_id}") # "gpt-4" - print(f"Model Name: {evaluation.model_name}") # "GPT-4" - print(f"Model Key: {evaluation.model_key}") # Internal key - print(f"Model Company: {evaluation.model_company}") # "OpenAI" -``` ## Benchmarks ### `get(type=None, name=None, timeout=None)` -Retrieves a list of available benchmarks with optional filtering parameters. +Retrieves a list of available benchmarks with optional filtering parameters. Both the `Atlas` and `AsyncAtlas` clients have this method. #### Parameters @@ -174,36 +87,7 @@ Retrieves a list of available benchmarks with optional filtering parameters. Returns a `List[Benchmark]` containing available benchmarks that match the filter criteria. Returns `None` if no benchmarks are found or if there's an error. -#### Examples - -##### Get All Benchmarks - -```python -from atlas import Atlas - -client = Atlas() - -# Get all available benchmarks (both custom and public) -benchmarks = client.benchmarks.get() - -if benchmarks: - print(f"Found {len(benchmarks)} benchmarks:") - for benchmark in benchmarks: - print(f" {benchmark.id} - {benchmark.name}") -else: - print("No benchmarks available") -``` - -##### Search by Name - -```python -# Search for benchmarks with "mmlu" in the name -mmlu_benchmark = client.benchmarks.get(name=["mmlu"])[0] - -if mmlu_benchmark: - print("MMLU benchmark found:") - print(f" {mmlu_benchmark.id} - {mmlu_benchmark.name}") -``` +Returns `Optional[List[Benchmark]]` - a list of `Benchmark` objects that match the filter criteria. Returns an empty list `[]` if no benchmarks match the criteria, or `None` if there's an error. #### Benchmark Object Properties @@ -213,118 +97,5 @@ Each `Benchmark` object in the returned list contains: | Property | Type | Description | |----------|------|-------------| | `id` | `str` | Unique benchmark identifier for use in evaluations | -| `name` | `str` | Human-readable benchmark name | -| `type` | `str` | Benchmark type ("custom" or "public") | -| `description` | `str` | Description of what the benchmark tests | - -#### Error Handling - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - benchmarks = client.benchmarks.get() - if benchmarks: - print(f"Retrieved {len(benchmarks)} benchmarks") - else: - print("No benchmarks available or error occurred") -except atlas.AuthenticationError: - print("Authentication failed - check your API key") -except atlas.PermissionDeniedError: - print("No permission to access benchmarks") -except atlas.APIConnectionError as e: - print(f"Connection error: {e}") -except atlas.APIError as e: - print(f"API error: {e}") -``` - -## Discovery and Validation - -### Finding Available Models and Benchmarks - -#### Check the Atlas Dashboard -The most reliable way to find available models and benchmarks: - -1. Log into your Atlas dashboard -2. Navigate to the evaluation creation page -3. View dropdown lists of available models and benchmarks - - -### Benchmark Identification - -Benchmarks are identified by string IDs representing different evaluation datasets: - -```python -from atlas import Atlas - -client = Atlas() - -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" # Benchmark ID -) -``` - -### Benchmark Information - -Evaluation responses include benchmark details: - -```python -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") - -if evaluation: - print(f"Dataset ID: {evaluation.dataset_id}") # "mmlu" - print(f"Dataset Name: {evaluation.dataset_name}") # "MMLU" -``` - - -## Troubleshooting - -### Model or Benchmark Not Found - -```python -try: - evaluation = client.evaluations.create( - model="nonexistent-model", - benchmark="mmlu" - ) -except atlas.NotFoundError: - print("Model or benchmark not found. Check:") - print("1. Spelling of model/benchmark ID") - print("2. Available options in Atlas dashboard") - print("3. Your organization's access permissions") -``` - -### Permission Issues - -```python -try: - evaluation = client.evaluations.create( - model="restricted-model", - benchmark="private-benchmark" - ) -except atlas.PermissionDeniedError: - print("Access denied. Possible causes:") - print("1. Model requires higher permission level") - print("2. Benchmark is not available to your organization") - print("3. Project doesn't have access to these resources") -``` - -### Validation Errors - -```python -try: - evaluation = client.evaluations.create( - model="", # Empty string - benchmark="mmlu" - ) -except atlas.BadRequestError: - print("Invalid request parameters:") - print("- Model and benchmark IDs cannot be empty") - print("- IDs must be valid strings") -``` - -For more information about available models and benchmarks, consult your Atlas dashboard or contact your LayerLens administrator. \ No newline at end of file +| `key` | `str` | Unique benchmark key/identifier that is similar to the name | +| `name` | `str` | Human-readable benchmark name | \ No newline at end of file From e793993cf9e0da857a88e3078215a98282def977 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 16:46:45 -0400 Subject: [PATCH 08/15] update evaluations docs --- SUMMARY.md | 8 +- docs/api-reference/evaluations.md | 353 +++++++++--------------- docs/api-reference/models-benchmarks.md | 4 +- 3 files changed, 138 insertions(+), 227 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index 6e9eaac..8de587e 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -2,15 +2,13 @@ * [Layerlens Python SDK Documentation](docs/README.md) * [Api-reference](docs/api-reference/README.md) * [Client Configuration](docs/api-reference/client.md) - * [Error Handling](docs/api-reference/errors.md) - * [Evaluations](docs/api-reference/evaluations.md) * [Models & Benchmarks](docs/api-reference/models-benchmarks.md) + * [Evaluations](docs/api-reference/evaluations.md) * [Results](docs/api-reference/results.md) + * [Error Handling](docs/api-reference/errors.md) * [Examples](docs/examples/README.md) - * [Creating Evaluations](docs/examples/creating-evaluations.md) + * [Running Evaluations](docs/examples/creating-evaluations.md) * [Retrieving Results](docs/examples/retrieving-results.md) - * [Working with Timeouts](docs/examples/timeouts.md) * [Troubleshooting](docs/troubleshooting/README.md) * [Authentication Problems](docs/troubleshooting/authentication.md) - * [Common Issues](docs/troubleshooting/common-issues.md) * [Error Codes Reference](docs/troubleshooting/error-codes.md) diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index 81649af..750b385 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -1,13 +1,103 @@ # Evaluations -The `evaluations` resource allows you to create and manage AI model evaluations against various benchmarks. This is the core functionality of the Atlas platform. +The `evaluations` resource on the atlas client allows you to create and manage evaluations against various benchmarks for your organization on the atlas platform. This is one of the core functionalities of the Atlas platform. ## Overview -An evaluation runs a specified model against a benchmark dataset and returns comprehensive metrics including accuracy, readability, toxicity, and ethics scores. +An evaluation runs a specified model against a benchmark dataset and returns comprehensive metrics. + + +The below example trigger evaluations using `gpt-4o` against `simpleQA`. + +> Before running the below examples ensure the model and benchmark being run are present on your organiztion. + +### Using Synchronous Client + +Below is an example showing how to trigger an evaluation, waiting for it to complete and finally fetching the evaluations results. + +```python +from atlas import Atlas + +# Construct sync client (API key from env or inline) +client = Atlas() + +# --- Models +models = client.models.get(type="public", name="gpt-4o") + +if not models: + print("gpt-4o not found, exiting") + +model = models[0] + +# --- Benchmarks +benchmarks = client.benchmarks.get(type="public", name="simpleQA") + +if not benchmarks: + print("SimpleQA benchmark not found, exiting") + +benchmark = benchmarks[0] + +# --- Create evaluation +evaluation = client.evaluations.create( + model=model, + benchmark=benchmark, +) + +# --- Wait for completion +evaluation = client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + timeout_seconds=600, # 10 minutes +) + +# --- Results +if evaluation.is_success: + # Loads the first page of results + results = client.results.get(evaluation=evaluation) + print("Results:", results) + +``` + +### Using Async Client + +```python +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get(type="public", name="gpt-4o") + model = models[0] + + # --- Benchmarks + benchmarks = await client.benchmarks.get(type="public", name="simpleQA") + benchmark = benchmarks[0] + + + # --- Create evaluation + evaluation = await client.evaluations.create(model=model, benchmark=benchmark) + + + await evaluation.wait_for_completion_async(interval_seconds=10) + + # --- Results + if evaluation.is_success: + results = await evaluation.get_results_async() + + +if __name__ == "__main__": + asyncio.run(main()) +``` ## Methods +Both the `Atlas` (synchronous) and `AsyncAtlas` (asynchronous) clients support the following methods. + ### `create(model, benchmark, timeout=None)` Creates a new evaluation for the specified model and benchmark. @@ -16,44 +106,30 @@ Creates a new evaluation for the specified model and benchmark. | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `model` | `str` | Yes | The model identifier to evaluate | -| `benchmark` | `str` | Yes | The benchmark dataset identifier | +| `model` | `Model` | Yes | The model to evaluate | +| `benchmark` | `Benchmark` | Yes | The benchmark to evaluate | | `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | #### Returns Returns an `Evaluation` object if successful, `None` if the evaluation could not be created. -#### Example +### `wait_for_completion(evaluation, interval_seconds=30, timeout_seconds=None)` -```python -from atlas import Atlas +Polls an evaluation until it completes (success, failure, or timeout) or the specified timeout is reached. -client = Atlas() +#### Parameters -# Create a basic evaluation -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" -) +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `evaluation` | `Evaluation` | Yes | The evaluation object to monitor | +| `interval_seconds` | `int` | No | Polling interval in seconds (default: 30) | +| `timeout_seconds` | `int \| None` | No | Maximum time to wait in seconds (no limit if None) | -if evaluation: - print(f"Evaluation created: {evaluation.id}") - print(f"Status: {evaluation.status}") -else: - print("Failed to create evaluation") -``` +#### Returns -#### With Custom Timeout +Returns the updated `Evaluation` object when completed, or `None` if polling fails. -```python -# Create evaluation with custom timeout (5 minutes) -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu", - timeout=300.0 -) -``` ### `get_by_id(evaluation_id, timeout=None)` @@ -80,24 +156,6 @@ client = Atlas() # Retrieve an evaluation by ID evaluation_id = "eval_abc123xyz" evaluation = client.evaluations.get_by_id(evaluation_id) - -if evaluation: - print(f"Found evaluation: {evaluation.id}") - print(f"Status: {evaluation.status}") - print(f"Model: {evaluation.model_name}") - print(f"Benchmark: {evaluation.dataset_name}") -else: - print(f"Evaluation {evaluation_id} not found") -``` - -#### With Custom Timeout - -```python -# Retrieve evaluation with custom timeout -evaluation = client.evaluations.get_by_id( - "eval_abc123xyz", - timeout=30.0 # 30 seconds -) ``` #### Async Usage @@ -121,197 +179,52 @@ async def get_evaluation(): asyncio.run(get_evaluation()) ``` -## Response Object +### `get_many(page=None, page_size=None, timeout=None)` -The `create` method returns an `Evaluation` object with the following properties: +Retrieves multiple evaluations with optional pagination support. -### Core Properties +#### Parameters -| Property | Type | Description | -|----------|------|-------------| -| `id` | `str` | Unique evaluation identifier | -| `status` | `str` | Current evaluation status | -| `status_description` | `str` | Detailed status description | -| `submitted_at` | `int` | Unix timestamp when evaluation was submitted | -| `finished_at` | `int` | Unix timestamp when evaluation finished | +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of evaluations per page (default: 100, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | -### Model Information +#### Returns -| Property | Type | Description | -|----------|------|-------------| -| `model_id` | `str` | Model identifier used in the request | -| `model_name` | `str` | Human-readable model name | -| `model_key` | `str` | Internal model key | -| `model_company` | `str` | Company that created the model | +Returns an `EvaluationsResponse` object containing: +- `evaluations`: List of `Evaluation` objects +- `pagination`: Pagination metadata with `page`, `page_size`, `total_pages`, and `total_count` -### Benchmark Information +Returns `None` if the request fails. -| Property | Type | Description | -|----------|------|-------------| -| `dataset_id` | `str` | Benchmark identifier used in the request | -| `dataset_name` | `str` | Human-readable benchmark name | +## Response Objects -### Performance Metrics +The `create`, `get_by_id` and `get_many` method returns an `Evaluation` objects with the following properties: -These properties are available once the evaluation is completed: +### Evaluation Object Properties | Property | Type | Description | |----------|------|-------------| -| `accuracy` | `float` | Overall accuracy score (0.0 to 1.0) | -| `readability_score` | `float` | Readability assessment score | -| `toxicity_score` | `float` | Toxicity assessment score | -| `ethics_score` | `float` | Ethics assessment score | +| `id` | `str` | Unique evaluation identifier | +| `status` | `EvaluationStatus` | Current evaluation status (enum) | +| `submitted_at` | `int` | Unix timestamp when evaluation was submitted | +| `finished_at` | `int` | Unix timestamp when evaluation finished | +| `model_id` | `str` | ID of the model used in the evaluation | +| `benchmark_id` | `str` | ID of the benchmark used (aliased as "dataset_id" in API) | | `average_duration` | `int` | Average response time in milliseconds | +| `accuracy` | `float` | Overall accuracy score (0.0 to 1.0) | + ## Evaluation Status -The `status` field can have the following values: +The `status` field is an `EvaluationStatus` enum with the following values: | Status | Description | |--------|-------------| | `"pending"` | Evaluation queued but not yet started | -| `"running"` | Evaluation currently in progress | -| `"completed"` | Evaluation finished successfully | -| `"failed"` | Evaluation failed due to an error | -| `"cancelled"` | Evaluation was cancelled by user | - -## Complete Example - -```python -import time -from atlas import Atlas -import atlas - -def create_and_monitor_evaluation(): - client = Atlas() - - try: - # Create evaluation - evaluation = client.evaluations.create( - model="gpt-3.5-turbo", - benchmark="mmlu" - ) - - if not evaluation: - print("❌ Failed to create evaluation") - return None - - print(f"✅ Evaluation created: {evaluation.id}") - print(f"📊 Model: {evaluation.model_name} ({evaluation.model_company})") - print(f"📋 Benchmark: {evaluation.dataset_name}") - print(f"⏰ Submitted at: {evaluation.submitted_at}") - print(f"🔄 Status: {evaluation.status}") - - # Note: In practice, you'd use webhooks or polling to check status - # This is just for demonstration - if evaluation.status == "completed": - print(f"\n📈 Results:") - print(f" Accuracy: {evaluation.accuracy:.2%}") - print(f" Readability: {evaluation.readability_score:.2f}") - print(f" Toxicity: {evaluation.toxicity_score:.2f}") - print(f" Ethics: {evaluation.ethics_score:.2f}") - print(f" Avg Duration: {evaluation.average_duration}ms") - - return evaluation - - except atlas.AuthenticationError: - print("❌ Authentication failed - check your API key") - except atlas.PermissionDeniedError: - print("❌ Permission denied - check your organization/project access") - except atlas.NotFoundError: - print("❌ Model or benchmark not found") - except atlas.RateLimitError: - print("❌ Rate limit exceeded - please wait and try again") - except atlas.APIConnectionError as e: - print(f"❌ Connection error: {e}") - except atlas.APIError as e: - print(f"❌ API error: {e}") - - return None - -if __name__ == "__main__": - evaluation = create_and_monitor_evaluation() -``` - - -## Error Handling - -### Common Errors - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - evaluation = client.evaluations.create( - model="nonexistent-model", - benchmark="mmlu" - ) -except atlas.NotFoundError: - print("Model 'nonexistent-model' not found") -except atlas.BadRequestError: - print("Invalid request parameters") -except atlas.UnprocessableEntityError: - print("Request parameters are valid but cannot be processed") -``` - -### Timeout Handling - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu", - timeout=30.0 # 30 seconds - ) -except atlas.APITimeoutError: - print("Request timed out - try increasing timeout or check network") -``` - -## Best Practices - -### 1. Check Return Values -```python -# ✅ Good - always check if evaluation was created -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -if evaluation: - print(f"Success: {evaluation.id}") -else: - print("Failed to create evaluation") - -# ❌ Bad - assuming success -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -print(f"Success: {evaluation.id}") # Could raise AttributeError -``` - -### 2. Handle Long-Running Operations -```python -# ✅ Good - appropriate timeout for evaluation creation -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu", - timeout=120.0 # 2 minutes -) - -# ❌ Bad - timeout too short -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu", - timeout=5.0 # Likely to timeout -) -``` - - - -## Next Steps - -- Learn how to [retrieve results](results.md) for your evaluations -- Explore [code examples](../examples/creating-evaluations.md) for common patterns -- Understand [error handling](errors.md) for robust applications \ No newline at end of file +| `"in-progress"` | Evaluation currently in progress | +| `"paused"` | Evaluation has been paused | +| `"success"` | Evaluation finished successfully | +| `"failure"` | Evaluation failed due to an error | diff --git a/docs/api-reference/models-benchmarks.md b/docs/api-reference/models-benchmarks.md index 855dab1..ba6a433 100644 --- a/docs/api-reference/models-benchmarks.md +++ b/docs/api-reference/models-benchmarks.md @@ -4,9 +4,9 @@ This page provides reference information about accessing the available models an ## Overview -Running evaluations on the atlas platform require a model and benchmark to be selected. +Running evaluations on the atlas platform require a model and benchmark to be selected. The availability of models and benchmarks depends on your organizations configuration. -The availability of models and benchmarks depends on your organizations configuration. +> Before running the below examples ensure the model and benchmark being run are present on your organiztion. ### Finding Available Models and Benchmarks From 82feca45a3f852ec3224e45a330fa3ece3bc3b35 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 16:50:18 -0400 Subject: [PATCH 09/15] removed unused getting started section --- docs/getting-started/README.md | 2 - docs/getting-started/authentication.md | 156 -------------------- docs/getting-started/installation.md | 55 ------- docs/getting-started/quickstart.md | 195 ------------------------- 4 files changed, 408 deletions(-) delete mode 100644 docs/getting-started/README.md delete mode 100644 docs/getting-started/authentication.md delete mode 100644 docs/getting-started/installation.md delete mode 100644 docs/getting-started/quickstart.md diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md deleted file mode 100644 index c453564..0000000 --- a/docs/getting-started/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Getting-started - diff --git a/docs/getting-started/authentication.md b/docs/getting-started/authentication.md deleted file mode 100644 index 61adcc5..0000000 --- a/docs/getting-started/authentication.md +++ /dev/null @@ -1,156 +0,0 @@ -# Authentication & Configuration - -The Atlas Python SDK uses API key authentication to securely access the LayerLens Atlas API. This guide covers how to set up authentication and configure your client. - -## Required Credentials - -You need three pieces of information to use the Atlas SDK: - -1. **API Key** - Your secret API key for authentication -2. **Organization ID** - Your organization identifier -3. **Project ID** - The project you want to work with - -## Getting Your Credentials - -1. **Log in to LayerLens Atlas**: Visit the LayerLens Atlas dashboard -2. **Navigate to Settings**: Go to your account or organization settings -3. **Generate API Key**: Create a new API key if you don't have one -4. **Copy IDs**: Note your Organization ID and Project ID from the dashboard - -## Environment Variables (Recommended) - -The most secure way to configure authentication is using environment variables: - -### Setting Environment Variables - -**Linux/macOS:** - -```bash -export LAYERLENS_ATLAS_API_KEY="your_api_key_here" -``` - -**Windows (Command Prompt):** - -```cmd -set LAYERLENS_ATLAS_API_KEY=your_api_key_here -``` - -**Windows (PowerShell):** - -```powershell -$env:LAYERLENS_ATLAS_API_KEY="your_api_key_here" -``` - -### Using a `.env` File - -Create a `.env` file in your project root: - -```bash -LAYERLENS_ATLAS_API_KEY=your_api_key_here -``` - -Then load it in your Python code: - -```python -from dotenv import load_dotenv -import os -from atlas import Atlas - -# Load environment variables from .env file -load_dotenv() - -# Client will automatically use environment variables -client = Atlas() -``` - -> **⚠️ Security Note**: Never commit `.env` files to version control. Add `.env` to your `.gitignore` file. - -## Client Configuration - -### Automatic Configuration - -When environment variables are set, the client configures itself automatically: - -```python -from atlas import Atlas - -# Uses environment variables automatically -client = Atlas() -``` - -### Explicit Configuration - -You can also pass credentials directly to the client: - -```python -from atlas import Atlas - -client = Atlas(api_key="your_api_key_here") -``` - -### Mixed Configuration - -You can mix environment variables with explicit parameters: - -```python -import os -from atlas import Atlas - -client = Atlas(api_key=os.environ.get("LAYERLENS_ATLAS_API_KEY")) -``` - -## Advanced Configuration - -### Timeout Configuration - -Configure request timeouts: - -```python -from atlas import Atlas -import httpx - -# Simple timeout (10 seconds) -client = Atlas(timeout=10.0) - -# Advanced timeout configuration -client = Atlas( - timeout=httpx.Timeout( - connect=5.0, # Connection timeout - read=30.0, # Read timeout - write=10.0, # Write timeout - pool=2.0 # Pool timeout - ) -) -``` - -## Validation - -The SDK will validate your configuration on first use: - -```python -from atlas import Atlas - -try: - client = Atlas() - # Test the connection - evaluation = client.evaluations.create(model="test", benchmark="test") -except atlas.AuthenticationError: - print("Invalid API key or authentication failed") -except atlas.PermissionDeniedError: - print("Valid API key but insufficient permissions") -except atlas.AtlasError as e: - print(f"Configuration error: {e}") -``` - -## Security Best Practices - -1. **Never hardcode credentials** in your source code -2. **Use environment variables** or secure credential management systems -3. **Rotate API keys regularly** for enhanced security -4. **Use different API keys** for different environments (dev, staging, prod) -5. **Monitor API key usage** in the LayerLens dashboard -6. **Revoke unused keys** immediately - -## Next Steps - -Once authentication is configured, proceed to the [Quick Start Guide](quickstart.md) to make your first API call. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md deleted file mode 100644 index 37e75e5..0000000 --- a/docs/getting-started/installation.md +++ /dev/null @@ -1,55 +0,0 @@ -# Installation - -The Atlas Python SDK supports Python 3.8 and above. You can install it using pip or your preferred Python package manager. - -## Install from PyPI - -```bash -pip install atlas -``` - -## Verify Installation - -After installation, verify that the SDK is working correctly: - -```python -import atlas -print(atlas.__version__) -``` - -This should print the version number of the installed SDK. - -## System Requirements - -- **Python**: 3.8 or higher -- **Operating Systems**: Windows, macOS, Linux -- **Dependencies**: The SDK automatically installs required dependencies: - - `httpx` - HTTP client library - - `pydantic` - Data validation and serialization - - `typing-extensions` - Enhanced type hints for older Python versions - -## Virtual Environment (Recommended) - -We strongly recommend using a virtual environment to avoid dependency conflicts: - -```bash -# Create virtual environment -python -m venv atlas-env - -# Activate it (Linux/macOS) -source atlas-env/bin/activate - -# Activate it (Windows) -atlas-env\Scripts\activate - -# Install the SDK -pip install atlas -``` - -## Upgrading - -To upgrade to the latest version: - -```bash -pip install --upgrade atlas -``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md deleted file mode 100644 index 1c34cde..0000000 --- a/docs/getting-started/quickstart.md +++ /dev/null @@ -1,195 +0,0 @@ -# Quick Start Guide - -This guide will help you make your first API call with the Atlas Python SDK. We'll walk through creating an evaluation and retrieving results. - -## Prerequisites - -Before you begin, ensure you have: - -1. ✅ [Installed the Atlas SDK](installation.md) -2. ✅ [Configured authentication](authentication.md) with your API key, organization ID, and project ID -3. ✅ Access to LayerLens Atlas platform - -## Your First Evaluation - -Let's create a simple evaluation to test your setup: - -```python -import os -from atlas import Atlas - -# Initialize the client (uses environment variables) -client = Atlas( - api_key=os.environ.get("LAYERLENS_ATLAS_API_KEY"), -) - -# Create an evaluation -evaluation = client.evaluations.create( - model="gpt-3.5-turbo", # Replace with your model ID - benchmark="mmlu" # Replace with your benchmark ID -) - -if evaluation: - print(f"✅ Evaluation created successfully!") - print(f" ID: {evaluation.id}") - print(f" Status: {evaluation.status}") - print(f" Model: {evaluation.model_name}") - print(f" Benchmark: {evaluation.dataset_name}") -else: - print("❌ Failed to create evaluation") -``` - -## Understanding the Response - -A successful evaluation creation returns an `Evaluation` object with the following key properties: - -```python -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") - -print(f"Evaluation ID: {evaluation.id}") -print(f"Status: {evaluation.status}") -print(f"Status Description: {evaluation.status_description}") -print(f"Submitted At: {evaluation.submitted_at}") -print(f"Model: {evaluation.model_name} ({evaluation.model_company})") -print(f"Dataset: {evaluation.dataset_name}") - -# Available when evaluation is completed -if evaluation.status == "completed": - print(f"Accuracy: {evaluation.accuracy}") - print(f"Readability Score: {evaluation.readability_score}") - print(f"Toxicity Score: {evaluation.toxicity_score}") - print(f"Ethics Score: {evaluation.ethics_score}") -``` - -## Retrieving Results - -Once your evaluation is complete, you can retrieve detailed results: - -```python -# Wait for evaluation to complete, then get results -if evaluation and evaluation.status == "completed": - results = client.results.get(evaluation_id=evaluation.id) - - if results: - print(f"📊 Retrieved {len(results)} results") - - # Examine the first result - first_result = results[0] - print(f"\nFirst Result:") - print(f" Subset: {first_result.subset}") - print(f" Prompt: {first_result.prompt[:100]}...") # First 100 chars - print(f" Model Output: {first_result.result[:100]}...") - print(f" Expected Answer: {first_result.truth}") - print(f" Score: {first_result.score}") - print(f" Duration: {first_result.duration}") - print(f" Metrics: {first_result.metrics}") -``` - -## Complete Example - -Here's a complete example that creates an evaluation and waits for results: - -```python -import os -import time -from atlas import Atlas - -def main(): - # Initialize client - client = Atlas() - - print("🚀 Creating evaluation...") - - try: - # Create evaluation - evaluation = client.evaluations.create( - model="gpt-3.5-turbo", - benchmark="mmlu" - ) - - if not evaluation: - print("❌ Failed to create evaluation") - return - - print(f"✅ Evaluation created: {evaluation.id}") - print(f" Status: {evaluation.status}") - - # Poll for completion (in a real app, use webhooks instead) - print("\n⏳ Waiting for evaluation to complete...") - - while evaluation.status not in ["completed", "failed", "cancelled"]: - time.sleep(30) # Wait 30 seconds - - # In practice, you'd re-fetch the evaluation status - # This is a simplified example - print(f" Status: {evaluation.status}") - - if evaluation.status == "completed": - print(f"🎉 Evaluation completed!") - print(f" Accuracy: {evaluation.accuracy:.2%}") - - # Get detailed results - results = client.results.get(evaluation_id=evaluation.id) - print(f"📊 Retrieved {len(results) if results else 0} detailed results") - - else: - print(f"❌ Evaluation failed with status: {evaluation.status}") - - except Exception as e: - print(f"❌ Error: {e}") - -if __name__ == "__main__": - main() -``` - -## Error Handling - -Always wrap your API calls in try-catch blocks: - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" - ) -except atlas.AuthenticationError: - print("❌ Authentication failed. Check your API key.") -except atlas.PermissionDeniedError: - print("❌ Permission denied. Check your organization/project access.") -except atlas.RateLimitError: - print("❌ Rate limit exceeded. Please wait and try again.") -except atlas.APIConnectionError as e: - print(f"❌ Connection error: {e}") -except atlas.APIStatusError as e: - print(f"❌ API error: {e.status_code} - {e}") -except Exception as e: - print(f"❌ Unexpected error: {e}") -``` - -## Available Models and Benchmarks - -To see available models and benchmarks, you can: - -1. **Check the LayerLens Atlas dashboard** for the most up-to-date list -2. **Contact support** for specific model or benchmark IDs - -## What's Next? - -Now that you've successfully made your first API call: - -1. **[Explore the API Reference](../api-reference/)** - Learn about all available methods -2. **[Check out Code Examples](../examples/)** - See practical usage patterns -3. **[Review Error Handling](../api-reference/errors.md)** - Handle edge cases gracefully -4. **[Security Best Practices](../security/)** - Secure your API usage - -## Need Help? - -- **Documentation**: Browse the complete [API Reference](../api-reference/) -- **Examples**: Check out more [Code Examples](../examples/) -- **Support**: Contact LayerLens support through your dashboard for technical assistance -- **Status**: Check [status.layerlens.com](https://status.layerlens.com) for service updates From 2a171281e72e90d019a2a7ae144973986fe28729 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 17:30:18 -0400 Subject: [PATCH 10/15] update evaluations docs to show results loading methods --- SUMMARY.md | 1 - docs/api-reference/evaluations.md | 226 +++++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 2 deletions(-) diff --git a/SUMMARY.md b/SUMMARY.md index 8de587e..7c6e16a 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -4,7 +4,6 @@ * [Client Configuration](docs/api-reference/client.md) * [Models & Benchmarks](docs/api-reference/models-benchmarks.md) * [Evaluations](docs/api-reference/evaluations.md) - * [Results](docs/api-reference/results.md) * [Error Handling](docs/api-reference/errors.md) * [Examples](docs/examples/README.md) * [Running Evaluations](docs/examples/creating-evaluations.md) diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index 750b385..53eab05 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -25,7 +25,7 @@ client = Atlas() models = client.models.get(type="public", name="gpt-4o") if not models: - print("gpt-4o not found, exiting") + print("gpt-4o not found") model = models[0] @@ -199,6 +199,226 @@ Returns an `EvaluationsResponse` object containing: Returns `None` if the request fails. +### `get_all_results(timeout=None)` + +Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a list of `Result` objects containing all results for the evaluation. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get an evaluation +evaluation = client.evaluations.get_by_id("eval_12345") + +if evaluation: + # Fetch all results (handles pagination automatically) + all_results = evaluation.get_all_results() +``` + +{{ ... }} + +| `"success"` | Evaluation finished successfully | +| `"failure"` | Evaluation failed due to an error | + +## Evaluation Instance Methods + +Once you have an `Evaluation` object (from `create()`, `get_by_id()`, etc.), you can call these methods directly on the evaluation instance to fetch its results. + +### `evaluation.get_results(page=None, page_size=None, timeout=None)` + +Fetches results for this evaluation with pagination support. This is a synchronous method that requires a sync client to be attached. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get an evaluation +evaluation = client.evaluations.get_by_id("eval_12345") + +if evaluation: + # Fetch first page of results + results_response = evaluation.get_results() + + if results_response: + print(f"Found {len(results_response.results)} results") + for result in results_response.results: + print(f"Score: {result.score}, Subset: {result.subset}") +``` + +### `evaluation.get_all_results(timeout=None)` + +Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method that requires a sync client to be attached. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a list of `Result` objects containing all results for the evaluation. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get an evaluation +evaluation = client.evaluations.get_by_id("eval_12345") + +if evaluation: + # Fetch all results (handles pagination automatically) + all_results = evaluation.get_all_results() + + print(f"Retrieved {len(all_results)} total results") + + # Calculate accuracy + correct_count = sum(1 for result in all_results if result.score > 0.5) + accuracy = correct_count / len(all_results) if all_results else 0 + print(f"Overall accuracy: {accuracy:.1%}") +``` + +### `get_all_results_async(timeout=None)` + +Asynchronously fetches all results for this evaluation by automatically handling pagination. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a list of `Result` objects containing all results for the evaluation. + +#### Example + +```python +from atlas import AsyncAtlas +import asyncio + +async def fetch_all_evaluation_results(): + client = AsyncAtlas() + + # Get an evaluation + evaluation = await client.evaluations.get_by_id("eval_12345") + + if evaluation: + # Fetch all results asynchronously (handles pagination automatically) + all_results = await evaluation.get_all_results_async() + + print(f"Retrieved {len(all_results)} total results") + + return all_results + + return None + +results = asyncio.run(fetch_all_evaluation_results()) +``` + +### `get_results(page=None, page_size=None, timeout=None)` + +Fetches results for this evaluation with pagination support. This is a synchronous method. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get evaluation first +evaluation = client.evaluations.get_by_id("eval_12345") +if not evaluation: + print("Evaluation not found") +else: + # + results_first_page = evaluation.get_results() +``` +### `get_results_async(page=None, page_size=None, timeout=None)` + +Asynchronously fetches results for this evaluation with pagination support. This requires an async client to be attached. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. + +#### Example + +```python +from atlas import AsyncAtlas +import asyncio + +async def fetch_evaluation_results(): + client = AsyncAtlas() + + # Get an evaluation + evaluation = await client.evaluations.get_by_id("eval_12345") + + if evaluation: + # Fetch first page of results asynchronously + first_page_results = await evaluation.get_results_async(page=1, page_size=50) + + if first_page_results: + return first_page_results + + return [] + +results = asyncio.run(fetch_evaluation_results()) +``` + ## Response Objects The `create`, `get_by_id` and `get_many` method returns an `Evaluation` objects with the following properties: @@ -228,3 +448,7 @@ The `status` field is an `EvaluationStatus` enum with the following values: | `"paused"` | Evaluation has been paused | | `"success"` | Evaluation finished successfully | | `"failure"` | Evaluation failed due to an error | + + +## Next Steps +- Explore [code examples](../examples/retrieving-results.md) for common analysis patterns From 3cacca016989c0b91b7a4a1e2d995f441edee34a Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Tue, 26 Aug 2025 17:38:56 -0400 Subject: [PATCH 11/15] remove duplicate evaluation docs --- docs/api-reference/evaluations.md | 146 +++++++++++------------------- 1 file changed, 52 insertions(+), 94 deletions(-) diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index 53eab05..67ca85c 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -199,47 +199,10 @@ Returns an `EvaluationsResponse` object containing: Returns `None` if the request fails. -### `get_all_results(timeout=None)` - -Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method - -#### Parameters - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | - -#### Returns - -Returns a list of `Result` objects containing all results for the evaluation. - -#### Example - -```python -from atlas import Atlas - -client = Atlas() - -# Get an evaluation -evaluation = client.evaluations.get_by_id("eval_12345") - -if evaluation: - # Fetch all results (handles pagination automatically) - all_results = evaluation.get_all_results() -``` - -{{ ... }} - -| `"success"` | Evaluation finished successfully | -| `"failure"` | Evaluation failed due to an error | -## Evaluation Instance Methods - -Once you have an `Evaluation` object (from `create()`, `get_by_id()`, etc.), you can call these methods directly on the evaluation instance to fetch its results. - -### `evaluation.get_results(page=None, page_size=None, timeout=None)` +### `get_results(page=None, page_size=None, timeout=None)` -Fetches results for this evaluation with pagination support. This is a synchronous method that requires a sync client to be attached. +Fetches results for this evaluation with pagination support. This is a synchronous method. #### Parameters @@ -260,22 +223,16 @@ from atlas import Atlas client = Atlas() -# Get an evaluation +# Get evaluation first evaluation = client.evaluations.get_by_id("eval_12345") - -if evaluation: - # Fetch first page of results - results_response = evaluation.get_results() - - if results_response: - print(f"Found {len(results_response.results)} results") - for result in results_response.results: - print(f"Score: {result.score}, Subset: {result.subset}") +if not evaluation: + print("Evaluation not found") +else: + # + results_first_page = evaluation.get_results() ``` -### `evaluation.get_all_results(timeout=None)` - -Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method that requires a sync client to be attached. +Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method #### Parameters @@ -300,28 +257,30 @@ evaluation = client.evaluations.get_by_id("eval_12345") if evaluation: # Fetch all results (handles pagination automatically) all_results = evaluation.get_all_results() - - print(f"Retrieved {len(all_results)} total results") - - # Calculate accuracy - correct_count = sum(1 for result in all_results if result.score > 0.5) - accuracy = correct_count / len(all_results) if all_results else 0 - print(f"Overall accuracy: {accuracy:.1%}") ``` -### `get_all_results_async(timeout=None)` +{{ ... }} -Asynchronously fetches all results for this evaluation by automatically handling pagination. +| `"success"` | Evaluation finished successfully | +| `"failure"` | Evaluation failed due to an error | + + + +### `get_results_async(page=None, page_size=None, timeout=None)` + +Asynchronously fetches results for this evaluation with pagination support. This requires an async client to be attached. #### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| +| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | | `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | #### Returns -Returns a list of `Result` objects containing all results for the evaluation. +Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. #### Example @@ -329,40 +288,36 @@ Returns a list of `Result` objects containing all results for the evaluation. from atlas import AsyncAtlas import asyncio -async def fetch_all_evaluation_results(): +async def fetch_evaluation_results(): client = AsyncAtlas() # Get an evaluation evaluation = await client.evaluations.get_by_id("eval_12345") if evaluation: - # Fetch all results asynchronously (handles pagination automatically) - all_results = await evaluation.get_all_results_async() - - print(f"Retrieved {len(all_results)} total results") + # Fetch first page of results asynchronously + first_page_results = await evaluation.get_results_async(page=1, page_size=50) - return all_results + if first_page_results: + return first_page_results - return None + return [] -results = asyncio.run(fetch_all_evaluation_results()) +results = asyncio.run(fetch_evaluation_results()) ``` -### `get_results(page=None, page_size=None, timeout=None)` - -Fetches results for this evaluation with pagination support. This is a synchronous method. +### `get_all_results(timeout=None)` +Fetches all results for this evaluation by automatically handling pagination. This is a synchronous method. #### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | -| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | | `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | #### Returns -Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. +Returns a list of `Result` objects containing all results for the evaluation. #### Example @@ -371,29 +326,30 @@ from atlas import Atlas client = Atlas() -# Get evaluation first +# Get an evaluation evaluation = client.evaluations.get_by_id("eval_12345") -if not evaluation: - print("Evaluation not found") -else: - # - results_first_page = evaluation.get_results() + +if evaluation: + # Fetch all results (handles pagination automatically) + all_results = evaluation.get_all_results() + + print(f"Retrieved {len(all_results)} total results") ``` -### `get_results_async(page=None, page_size=None, timeout=None)` -Asynchronously fetches results for this evaluation with pagination support. This requires an async client to be attached. + +### `get_all_results_async(timeout=None)` + +Asynchronously fetches all results for this evaluation by automatically handling pagination. #### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| `page` | `int \| None` | No | Page number for pagination (1-based, defaults to 1) | -| `page_size` | `int \| None` | No | Number of results per page (default: 100, max: 500) | | `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | #### Returns -Returns a `ResultsResponse` object containing results and pagination metadata, or `None` if the request fails. +Returns a list of `Result` objects containing all results for the evaluation. #### Example @@ -401,24 +357,26 @@ Returns a `ResultsResponse` object containing results and pagination metadata, o from atlas import AsyncAtlas import asyncio -async def fetch_evaluation_results(): +async def fetch_all_evaluation_results(): client = AsyncAtlas() # Get an evaluation evaluation = await client.evaluations.get_by_id("eval_12345") if evaluation: - # Fetch first page of results asynchronously - first_page_results = await evaluation.get_results_async(page=1, page_size=50) + # Fetch all results asynchronously (handles pagination automatically) + all_results = await evaluation.get_all_results_async() - if first_page_results: - return first_page_results + print(f"Retrieved {len(all_results)} total results") + + return all_results - return [] + return None -results = asyncio.run(fetch_evaluation_results()) +results = asyncio.run(fetch_all_evaluation_results()) ``` + ## Response Objects The `create`, `get_by_id` and `get_many` method returns an `Evaluation` objects with the following properties: From 2bda56ea285bd0c77fd912d5f7a7f99e3ab7e88e Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Wed, 27 Aug 2025 09:21:38 -0400 Subject: [PATCH 12/15] update examples page --- SUMMARY.md | 1 - docs/api-reference/evaluations.md | 2 +- docs/api-reference/results.md | 302 ++------------------- docs/examples/advanced-usage.md | 376 -------------------------- docs/examples/creating-evaluations.md | 253 +++++++++++------ docs/examples/retrieving-results.md | 138 ---------- docs/examples/timeouts.md | 231 ---------------- 7 files changed, 185 insertions(+), 1118 deletions(-) delete mode 100644 docs/examples/advanced-usage.md delete mode 100644 docs/examples/retrieving-results.md delete mode 100644 docs/examples/timeouts.md diff --git a/SUMMARY.md b/SUMMARY.md index 7c6e16a..17af81c 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -7,7 +7,6 @@ * [Error Handling](docs/api-reference/errors.md) * [Examples](docs/examples/README.md) * [Running Evaluations](docs/examples/creating-evaluations.md) - * [Retrieving Results](docs/examples/retrieving-results.md) * [Troubleshooting](docs/troubleshooting/README.md) * [Authentication Problems](docs/troubleshooting/authentication.md) * [Error Codes Reference](docs/troubleshooting/error-codes.md) diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index 67ca85c..46234b1 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -395,7 +395,7 @@ The `create`, `get_by_id` and `get_many` method returns an `Evaluation` objects | `accuracy` | `float` | Overall accuracy score (0.0 to 1.0) | -## Evaluation Status +#### Evaluation Status The `status` field is an `EvaluationStatus` enum with the following values: diff --git a/docs/api-reference/results.md b/docs/api-reference/results.md index ae02de1..59840fc 100644 --- a/docs/api-reference/results.md +++ b/docs/api-reference/results.md @@ -1,16 +1,19 @@ # Results -The `results` resource allows you to retrieve detailed results from completed evaluations. This provides granular insight into how your model performed on individual test cases. +The `results` resource allows you to retrieve detailed results from completed or partially completed evaluations. This provides granular insight into how your model performed on individual test cases. + ## Overview -Results contain detailed information about each test case in an evaluation, including the prompt, model response, expected answer, scoring metrics, and performance data. +Results contain detailed information about each test case in an evaluation. + +> Both the `Atlas` (synchronous) and `AsyncAtlas` (asynchronous) clients support the following methods. ## Methods ### `get_all(evaluation, timeout=None)` -Retrieves all results for a specific evaluation by automatically iterating through all pages. This is a convenience method that handles pagination internally. +Retrieves all results for a specific evaluation by automatically iterating through all pages. This is a convenience method that handles pagination through all results internally. #### Parameters @@ -62,7 +65,7 @@ async def get_all_results(): return all_results -# Run the async function +# Run the async fetching of results asyncio.run(get_all_results()) ``` @@ -90,30 +93,6 @@ client = Atlas() # Get all results directly by evaluation ID all_results = client.results.get_all_by_id(evaluation_id="eval_12345") - -if all_results: - print(f"Retrieved {len(all_results)} total results") - - # Calculate overall statistics - total_score = sum(result.score for result in all_results) - avg_score = total_score / len(all_results) - correct_count = sum(1 for result in all_results if result.score > 0.5) - accuracy = correct_count / len(all_results) - - print(f"Overall accuracy: {accuracy:.1%}") - print(f"Average score: {avg_score:.3f}") -else: - print("No results found for evaluation") -``` - -#### With Custom Timeout - -```python -# Get all results with custom timeout (5 minutes) -all_results = client.results.get_all_by_id( - evaluation_id="eval_12345", - timeout=300.0 -) ``` #### Async Usage @@ -147,18 +126,17 @@ Retrieves detailed results for a specific evaluation with optional pagination su | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `evaluation_id` | `str` | Yes | The evaluation identifier to get results for | -| `page` | `int \| None` | No | Page number for pagination (1-based). If not provided, returns first page or all results based on API default | -| `page_size` | `int \| None` | No | Number of results per page (default: 100). Maximum allowed may be limited by API | +| `page` | `int \| None` | No | Page number for pagination. If not provided, returns first page is returned by default | +| `page_size` | `int \| None` | No | Number of results per page (default: 100). Maximum allowed page_size is 500 | | `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | #### Returns -Returns a `ResultsData` object containing results, evaluation metadata, and pagination information if successful, `None` if no results are found or the evaluation doesn't exist. +Returns a `ResultsResponse` object containing results, evaluation metadata, and pagination information if successful, `None` if no results are found or the evaluation doesn't exist. -The `ResultsData` object includes: +The `ResultsResponse` object includes: - `results`: List of `Result` objects for the current page - `evaluation_id`: The evaluation ID -- `metrics`: Performance metrics including score ranges - `pagination`: Pagination metadata (total_count, page_size, total_pages) #### Examples @@ -240,17 +218,6 @@ while True: print("Finished processing all results") ``` -#### With Custom Timeout - -```python -# Get results with custom timeout (2 minutes) and pagination -results_data = client.results.get( - evaluation_id="eval_12345", - page=1, - page_size=50, - timeout=120.0 -) -``` ## Pagination Information @@ -321,258 +288,25 @@ Results can contain thousands of individual test cases. Consider using the async import asyncio from atlas import AsyncAtlas -async def fetch_results_efficiently(): - client = AsyncAtlas() - - # Get evaluation first - evaluation = await client.evaluations.get_by_id("eval_12345") - if not evaluation: - print("Evaluation not found") - return None - - # Good - async fetch with size awareness - results = await client.results.get_all(evaluation=evaluation) - if results: - print(f"Retrieved {len(results)} results asynchronously") - return results - else: - print("No results available") - return None - -# Run the async function -asyncio.run(fetch_results_efficiently()) -``` - -```python -# Bad - synchronous blocking call for large datasets -from atlas import Atlas - -client = Atlas() -results = client.results.get(evaluation_id="eval_12345") -# This blocks the entire thread while fetching thousands of results -``` - -### Manually pagination through results - -For evaluations with many test cases, use pagination to efficiently process results: - -```python -from atlas import Atlas - -def process_results_efficiently(evaluation_id: str, page_size: int = 100): - """Process large result sets using pagination""" - client = Atlas() - - # Get first page to understand total scope - first_page = client.results.get(evaluation_id=evaluation_id, page=1, page_size=page_size) - if not first_page: - print("No results found") - return - - total_count = first_page.pagination.total_count - total_pages = first_page.pagination.total_pages - - print(f"Processing {total_count} results across {total_pages} pages...") - - # Process each page - for page_num in range(1, total_pages + 1): - print(f"Processing page {page_num}/{total_pages}...") - - # Get current page (reuse first_page for page 1) - if page_num == 1: - results_data = first_page - else: - results_data = client.results.get( - evaluation_id=evaluation_id, - page=page_num, - page_size=page_size - ) - - if not results_data: - print(f"Failed to get page {page_num}") - continue - - # Process current page - for result in results_data.results: - # Your processing logic here - pass - - print(f"Completed page {page_num} ({len(results_data.results)} results)") - - print(f"Finished processing all {total_count} results") - -# Usage -process_results_efficiently("eval_12345", page_size=50) -``` - -## Performance Considerations - -### Large Result Sets -Results can contain thousands of individual test cases. Consider using the async client to load results asynchronously: - -```python -import asyncio -from atlas import AsyncAtlas - -async def fetch_results_efficiently(): - client = AsyncAtlas() +async def fetch_results_async(): + async_client = AsyncAtlas() # Get evaluation first - evaluation = await client.evaluations.get_by_id("eval_12345") + evaluation = await async_client.evaluations.get_by_id("eval_12345") if not evaluation: print("Evaluation not found") return None - # Good - async fetch with size awareness - results = await client.results.get_all(evaluation=evaluation) + # async results fetching all pages of results + results = await async_client.results.get_all(evaluation=evaluation) if results: - print(f"Retrieved {len(results)} results asynchronously") - if len(results) > 1000: - print("Large result set - consider processing in chunks") return results else: - print("No results available") return None # Run the async function -asyncio.run(fetch_results_efficiently()) -``` - -```python -# Bad - synchronous blocking call for large datasets -from atlas import Atlas - -client = Atlas() -results = client.results.get(evaluation_id="eval_12345") -# This blocks the entire thread while fetching thousands of results -``` - -## Filtering and Analysis - -### Filter by Subset - -```python -def analyze_subset_performance(results, target_subset): - subset_results = [r for r in results if r.subset == target_subset] - - if not subset_results: - print(f"No results found for subset '{target_subset}'") - return - - accuracy = sum(1 for r in subset_results if r.score > 0.5) / len(subset_results) - avg_duration = sum(r.duration for r in subset_results) / len(subset_results) - - print(f"Subset '{target_subset}' Performance:") - print(f" Test cases: {len(subset_results)}") - print(f" Accuracy: {accuracy:.1%}") - print(f" Average duration: {avg_duration}") - -# Usage -results = client.results.get(evaluation_id="eval_12345") -if results: - analyze_subset_performance(results, "elementary_mathematics") -``` - -### Find Difficult Cases - -```python -def find_difficult_cases(results, score_threshold=0.3): - """Find test cases where the model struggled""" - difficult_cases = [r for r in results if r.score < score_threshold] - - print(f"Found {len(difficult_cases)} difficult cases (score < {score_threshold})") - - for case in difficult_cases[:5]: # Show first 5 - print(f"\nDifficult Case [{case.subset}]:") - print(f" Prompt: {case.prompt[:100]}...") - print(f" Model: {case.result[:50]}...") - print(f" Expected: {case.truth[:50]}...") - print(f" Score: {case.score}") - -# Usage -results = client.results.get(evaluation_id="eval_12345") -if results: - find_difficult_cases(results) -``` - -## Error Handling - -### Common Errors - -```python -import atlas -from atlas import Atlas - -client = Atlas() - -try: - results = client.results.get(evaluation_id="nonexistent_eval") -except atlas.NotFoundError: - print("Evaluation not found or no results available") -except atlas.AuthenticationError: - print("Authentication failed") -except atlas.PermissionDeniedError: - print("No permission to access this evaluation") -``` - -### Handling Empty Results - -```python -def safe_get_results(client, evaluation_id): - """Safely get results with proper error handling""" - try: - results = client.results.get(evaluation_id=evaluation_id) - - if results is None: - print(f"No results found for evaluation {evaluation_id}") - print("This could mean:") - print("- Evaluation doesn't exist") - print("- Evaluation not yet completed") - print("- No permission to access results") - return [] - - if len(results) == 0: - print(f"Evaluation {evaluation_id} has no test cases") - return [] - - return results - - except atlas.APIError as e: - print(f"Error retrieving results: {e}") - return [] -``` - -## Best Practices - -### 1. Always Check for Results -```python -# Good - check if results exist -results = client.results.get(evaluation_id="eval_12345") -if results: - print(f"Found {len(results)} results") -else: - print("No results available") - -# Bad - assume results exist -results = client.results.get(evaluation_id="eval_12345") -print(f"Found {len(results)} results") # Could raise AttributeError -``` - -### 2. Handle Large Result Sets Appropriately -```python -# Good - process in chunks for large sets -if len(results) > 1000: - for i in range(0, len(results), 100): - chunk = results[i:i+100] - process_chunk(chunk) - -# Bad - process everything in memory -for result in results: # Could be thousands of results - expensive_processing(result) +asyncio.run(fetch_results_async()) ``` ## Next Steps - -- Learn about [error handling](errors.md) for robust applications -- Explore [code examples](../examples/retrieving-results.md) for common analysis patterns -- Check out [troubleshooting](../troubleshooting/) for common issues \ No newline at end of file +- Explore [code examples](../examples/retrieving-results.md) for common analysis patterns \ No newline at end of file diff --git a/docs/examples/advanced-usage.md b/docs/examples/advanced-usage.md deleted file mode 100644 index e880f7e..0000000 --- a/docs/examples/advanced-usage.md +++ /dev/null @@ -1,376 +0,0 @@ -# Advanced Usage - -Common patterns and best practices for using the Atlas Python SDK in production. - -## Environment Setup - -```python -# Set your API key as an environment variable -export LAYERLENS_ATLAS_API_KEY="your_api_key_here" - -# Then in your code: -from atlas import Atlas -client = Atlas() # Automatically reads from environment -``` - -## Async Operations - -### Basic Async Usage - -```python -import asyncio -from atlas import AsyncAtlas - -async def run_evaluation(): - # Create async client - client = AsyncAtlas() - - # Get models and benchmarks concurrently - models_task = client.models.get() - benchmarks_task = client.benchmarks.get() - models, benchmarks = await asyncio.gather(models_task, benchmarks_task) - - # Create evaluation - evaluation = await client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - # Wait for completion - completed = await client.evaluations.wait_for_completion(evaluation) - - if completed.is_success: - # Get results - results = await client.results.get_by_id(evaluation_id=completed.id) - return results - - return None - -# Run it -results = asyncio.run(run_evaluation()) -``` - -### Multiple Concurrent Evaluations - -```python -import asyncio -from atlas import AsyncAtlas - -async def create_multiple_evaluations(): - client = AsyncAtlas() - - models = await client.models.get() - benchmarks = await client.benchmarks.get() - - # Create multiple evaluations concurrently - tasks = [] - for model in models[:3]: # First 3 models - for benchmark in benchmarks[:2]: # First 2 benchmarks - task = client.evaluations.create(model=model, benchmark=benchmark) - tasks.append(task) - - # Wait for all to complete - evaluations = await asyncio.gather(*tasks) - - # Filter successful ones - successful = [e for e in evaluations if e is not None] - print(f"Created {len(successful)} evaluations") - - return successful - -# Run it -evaluations = asyncio.run(create_multiple_evaluations()) -``` - -## Error Handling - -### Basic Error Handling - -```python -from atlas import Atlas -import atlas - -def safe_create_evaluation(): - client = Atlas() - - try: - models = client.models.get() - benchmarks = client.benchmarks.get() - - evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - return evaluation - - except atlas.AuthenticationError: - print("Invalid API key") - return None - except atlas.NotFoundError: - print("Model or benchmark not found") - return None - except atlas.RateLimitError as e: - print(f"Rate limited. Try again in {e.retry_after} seconds") - return None - except atlas.APIError as e: - print(f"API error: {e}") - return None - -evaluation = safe_create_evaluation() -``` - -### Retry Logic - -```python -import time -from atlas import Atlas -import atlas - -def create_evaluation_with_retry(max_retries=3): - client = Atlas() - - for attempt in range(max_retries): - try: - models = client.models.get() - benchmarks = client.benchmarks.get() - - evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - print(f"Success on attempt {attempt + 1}") - return evaluation - - except atlas.RateLimitError as e: - if attempt < max_retries - 1: - wait_time = e.retry_after or (2 ** attempt) - print(f"Rate limited, waiting {wait_time}s...") - time.sleep(wait_time) - else: - print("Max retries exceeded") - return None - - except atlas.APIError as e: - if attempt < max_retries - 1: - print(f"Attempt {attempt + 1} failed: {e}") - time.sleep(2 ** attempt) # Exponential backoff - else: - print(f"Failed after {max_retries} attempts: {e}") - return None - - return None - -evaluation = create_evaluation_with_retry() -``` - -## Timeouts - -### Custom Timeouts - -```python -from atlas import Atlas - -# Different timeout strategies -quick_client = Atlas(timeout=30.0) # 30 seconds for testing -normal_client = Atlas(timeout=300.0) # 5 minutes for normal use -patient_client = Atlas(timeout=1800.0) # 30 minutes for long evaluations - -# Use appropriate client for your use case -evaluation = patient_client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] -) -``` - -### Per-Request Timeouts - -```python -from atlas import Atlas - -client = Atlas() - -# Override timeout for specific operations -models = client.models.get() -benchmarks = client.benchmarks.get() - -# Long evaluation with extended timeout -evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], - timeout=3600.0 # 1 hour timeout for this specific request -) -``` - -## Batch Processing - -### Process Multiple Evaluations - -```python -from atlas import Atlas - -def run_evaluation_batch(): - client = Atlas() - - models = client.models.get() - benchmarks = client.benchmarks.get() - - results = [] - - # Create evaluations - for model in models[:2]: - for benchmark in benchmarks[:2]: - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - print(f"Created: {model.id} + {benchmark.id} = {evaluation.id}") - results.append({ - 'model': model.id, - 'benchmark': benchmark.id, - 'evaluation_id': evaluation.id, - 'status': 'created' - }) - - # Small delay to avoid rate limits - time.sleep(1) - - except Exception as e: - print(f"Failed: {model.id} + {benchmark.id} - {e}") - results.append({ - 'model': model.id, - 'benchmark': benchmark.id, - 'error': str(e), - 'status': 'failed' - }) - - return results - -batch_results = run_evaluation_batch() -``` - -## Pagination Best Practices - -### Memory-Efficient Pagination - -```python -from atlas import Atlas - -def process_large_results(evaluation_id): - """Process large result sets without loading everything into memory""" - client = Atlas() - - page = 1 - total_processed = 0 - total_correct = 0 - - while True: - # Get one page at a time - results_data = client.results.get_by_id( - evaluation_id=evaluation_id, - page=page, - page_size=100 - ) - - if not results_data or not results_data.results: - break - - # Process this page - page_correct = sum(1 for r in results_data.results if r.score > 0.5) - total_correct += page_correct - total_processed += len(results_data.results) - - # Show progress - accuracy = total_correct / total_processed - print(f"Page {page}: {accuracy:.1%} accuracy ({total_processed:,} processed)") - - if page >= results_data.pagination.total_pages: - break - - page += 1 - - final_accuracy = total_correct / total_processed if total_processed > 0 else 0 - print(f"Final: {final_accuracy:.1%} accuracy ({total_processed:,} total)") - - return final_accuracy - -accuracy = process_large_results("your_evaluation_id") -``` - -## Health Checks - -### Simple Health Check - -```python -from atlas import Atlas -import atlas - -def check_atlas_health(): - """Check if Atlas API is reachable""" - try: - client = Atlas(timeout=10.0) - - # Try to get models (quick operation) - models = client.models.get() - - return { - 'status': 'healthy', - 'models_count': len(models) - } - - except atlas.AuthenticationError: - return {'status': 'unhealthy', 'error': 'authentication_failed'} - except atlas.APIConnectionError: - return {'status': 'unhealthy', 'error': 'connection_failed'} - except atlas.APITimeoutError: - return {'status': 'unhealthy', 'error': 'timeout'} - except Exception as e: - return {'status': 'unhealthy', 'error': str(e)} - -health = check_atlas_health() -if health['status'] == 'healthy': - print(f"Atlas is healthy ({health['models_count']} models available)") -else: - print(f"Atlas is unhealthy: {health['error']}") -``` - -## Monitoring and Logging - -### Basic Logging - -```python -import logging -from atlas import Atlas - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def create_evaluation_with_logging(): - client = Atlas() - - logger.info("Starting evaluation creation") - - try: - models = client.models.get() - benchmarks = client.benchmarks.get() - - logger.info(f"Found {len(models)} models, {len(benchmarks)} benchmarks") - - evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - logger.info(f"Created evaluation: {evaluation.id}") - return evaluation - - except Exception as e: - logger.error(f"Failed to create evaluation: {e}") - return None - -evaluation = create_evaluation_with_logging() -``` diff --git a/docs/examples/creating-evaluations.md b/docs/examples/creating-evaluations.md index ff4a935..9364602 100644 --- a/docs/examples/creating-evaluations.md +++ b/docs/examples/creating-evaluations.md @@ -1,110 +1,95 @@ # Creating Evaluations -Simple examples for creating AI model evaluations with the Atlas Python SDK. +Examples for creating evaluations on the Atlas platform using the Layerlens python sdk. -## Quick Start +> Before running the below examples ensure the model and benchmark being run are present on your organiztion. -### Basic Evaluation +## Basic Evaluation + +### Using Synchronous Client + +Below is an example showing how to trigger an evaluation, waiting for it to complete and finally fetching the evaluations results. ```python from atlas import Atlas -# Initialize client (reads LAYERLENS_ATLAS_API_KEY from environment) +# Construct sync client (API key from env or inline) client = Atlas() -# Get available models and benchmarks -models = client.models.get() -benchmarks = client.benchmarks.get() +# --- Models +models = client.models.get(type="public", name="gpt-4o") -print(f"Available: {len(models)} models, {len(benchmarks)} benchmarks") +if not models: + print("gpt-4o not found") -# Create evaluation with first available model and benchmark -evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] -) +model = models[0] -print(f"Created evaluation: {evaluation.id}") -print(f"Status: {evaluation.status}") -``` +# --- Benchmarks +benchmarks = client.benchmarks.get(type="public", name="simpleQA") -### Choose Specific Model and Benchmark +if not benchmarks: + print("SimpleQA benchmark not found, exiting") -```python -from atlas import Atlas +benchmark = benchmarks[0] -client = Atlas() +# --- Create evaluation +evaluation = client.evaluations.create( + model=model, + benchmark=benchmark, +) -# Get all available options -models = client.models.get() -benchmarks = client.benchmarks.get() +# --- Wait for completion +evaluation = client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + timeout_seconds=600, # 10 minutes +) -# Find specific model and benchmark -gpt4_model = next((m for m in models if "gpt-4" in m.id), None) -mmlu_benchmark = next((b for b in benchmarks if "mmlu" in b.id), None) +# --- Results +if evaluation.is_success: + # Loads the first page of results + results = client.results.get(evaluation=evaluation) + print("Results:", results) -if gpt4_model and mmlu_benchmark: - evaluation = client.evaluations.create( - model=gpt4_model, - benchmark=mmlu_benchmark - ) - print(f"Created: {evaluation.id}") -else: - print("Model or benchmark not found") - print(f"Available models: {[m.id for m in models[:5]]}...") - print(f"Available benchmarks: {[b.id for b in benchmarks[:5]]}...") ``` -## Async Version +### Using Async Client ```python import asyncio + from atlas import AsyncAtlas -async def create_evaluation(): + +async def main(): + # Construct async client client = AsyncAtlas() - - models = await client.models.get() - benchmarks = await client.benchmarks.get() - - evaluation = await client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - print(f"Created evaluation: {evaluation.id}") - return evaluation -# Run it -asyncio.run(create_evaluation()) -``` + # --- Models + models = await client.models.get(type="public", name="gpt-4o") + model = models[0] -## Wait for Completion + # --- Benchmarks + benchmarks = await client.benchmarks.get(type="public", name="simpleQA") + benchmark = benchmarks[0] -```python -from atlas import Atlas -client = Atlas() -models = client.models.get() -benchmarks = client.benchmarks.get() + # --- Create evaluation + evaluation = await client.evaluations.create(model=model, benchmark=benchmark) -# Create evaluation -evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) -print(f"Created: {evaluation.id}") -# Wait for it to complete (this may take several minutes) -completed_evaluation = client.evaluations.wait_for_completion( - evaluation, - interval_seconds=30, # Check every 30 seconds - timeout_seconds=1800 # 30 minute timeout -) + await evaluation.wait_for_completion_async(interval_seconds=10) + + # --- Results + if evaluation.is_success: + results = await evaluation.get_results_async() + -if completed_evaluation.is_success: - print("Evaluation completed successfully!") -else: - print(f"Evaluation failed: {completed_evaluation.status}") +if __name__ == "__main__": + asyncio.run(main()) ``` + ## Error Handling ```python @@ -121,7 +106,6 @@ try: model=models[0], benchmark=benchmarks[0] ) - print(f"Success: {evaluation.id}") except atlas.AuthenticationError: print("Check your API key") @@ -131,22 +115,117 @@ except atlas.APIError as e: print(f"API error: {e}") ``` -## Multiple Evaluations +## Triggering Multiple Evaluations ```python -from atlas import Atlas +import asyncio -client = Atlas() -models = client.models.get() -benchmarks = client.benchmarks.get() - -# Create multiple evaluations -evaluations = [] -for model in models[:3]: # First 3 models - for benchmark in benchmarks[:2]: # First 2 benchmarks - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - evaluations.append(evaluation) - print(f"Created: {model.id} + {benchmark.id} = {evaluation.id}") - -print(f"Created {len(evaluations)} evaluations total") +from atlas import AsyncAtlas + + +async def create_and_run_evaluation(client, model, benchmark, eval_number): + """Create and run a single evaluation, tracking progress.""" + try: + print(f"Starting evaluation #{eval_number}...") + + # Create evaluation + evaluation = await client.evaluations.create(model=model, benchmark=benchmark) + + # Wait for completion + evaluation = await client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + timeout_seconds=600, # 10 minutes + ) + + # Get results if successful + if evaluation.is_success: + results = await client.results.get_all(evaluation=evaluation) + return results + else: + return None + + except Exception as e: + print(f"✗ Error in evaluation #{eval_number}: {e}") + return eval_number, None, 0, False + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get() + + # --- Benchmarks + benchmarks = await client.benchmarks.get() + + # Use first model and benchmark for all evaluations + target_model = models[0] + target_benchmark = benchmarks[0] + + print(f"Using model: {target_model}") + print(f"Using benchmark: {target_benchmark}") + + # Create 3 evaluation tasks + num_evaluations = 3 + print(f"Starting {num_evaluations} evaluations in parallel...") + + tasks = [create_and_run_evaluation(client, target_model, target_benchmark, i + 1) for i in range(num_evaluations)] + + # Execute all evaluations concurrently + await asyncio.gather(*tasks, return_exceptions=True) + + +if __name__ == "__main__": + asyncio.run(main()) ``` + +## Fetching Results of Multiple Evaluations Async + +```python + +import asyncio + +from atlas import AsyncAtlas + + +async def fetch_evaluation_results(client, evaluation_id): + """Fetch results for a single evaluation and print when loaded.""" + try: + print(f"Fetching evaluation {evaluation_id}...") + evaluation = await client.evaluations.get_by_id(evaluation_id) + # Get all results for this evaluation + results = await client.results.get_all(evaluation=evaluation) + print(f"Loaded {len(results)} results for evaluation {evaluation_id}") + + return evaluation_id, results + except Exception as e: + print(f"Error fetching evaluation {evaluation_id}: {e}") + return evaluation_id, None + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # List of example evaluation IDs to fetch + + evaluation_ids = ["68a65a3de7ad047fbd8e7d4", "688a54c673f6b2835cc7278"] + + print(f"Starting async fetch for {len(evaluation_ids)} evaluations...") + + # Create tasks for concurrent execution + tasks = [fetch_evaluation_results(client, eval_id) for eval_id in evaluation_ids] + + # Execute all tasks concurrently and print results as they complete + results = await asyncio.gather(*tasks, return_exceptions=True) + + print("=" * 80) + print("Summary:") + successful = sum(1 for _, result in results if result is not None and not isinstance(result, Exception)) + +if __name__ == "__main__": + asyncio.run(main()) + +``` \ No newline at end of file diff --git a/docs/examples/retrieving-results.md b/docs/examples/retrieving-results.md deleted file mode 100644 index 07a7a2d..0000000 --- a/docs/examples/retrieving-results.md +++ /dev/null @@ -1,138 +0,0 @@ -# Retrieving Results - -Simple examples for getting evaluation results with the Atlas Python SDK. - -## Basic Result Retrieval - -### Get Results (First Page) - -```python -from atlas import Atlas - -client = Atlas() - -# Get results for a specific evaluation -evaluation_id = "your_evaluation_id" -results = client.results.get_by_id(evaluation_id=evaluation_id) - -if results: - print(f"Results for evaluation: {results.evaluation_id}") - print(f"Total results: {results.pagination.total_count:,}") - print(f"Showing page 1 of {results.pagination.total_pages}") - print(f"Results on this page: {len(results.results)}") - - # Show first few results - for i, result in enumerate(results.results[:3]): - print(f"\nResult {i+1}:") - print(f" Score: {result.score}") - print(f" Subset: {result.subset}") - print(f" Prompt: {result.prompt[:100]}...") - print(f" Response: {result.result[:50]}...") -else: - print("No results found") -``` - -## Get All Results Without Manual Pagination - -### Using get_all() Method - -The easiest way to retrieve all results is using the `get_all()` method, which handles pagination automatically: - -```python -import asyncio -from atlas import AsyncAtlas - -async def main(): - # Construct async client - client = AsyncAtlas() - - # --- Models - models = await client.models.get() - print(f"Found {len(models)} models") - - # --- Benchmarks - benchmarks = await client.benchmarks.get() - print(f"Found {len(benchmarks)} benchmarks") - - # --- Create evaluation - evaluation = await client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], - ) - print(f"Created evaluation {evaluation.id}, status={evaluation.status}") - - # --- Wait for completion - evaluation = await client.evaluations.wait_for_completion( - evaluation, - interval_seconds=10, - # Keep in mind that the evaluation will take a while to complete, so you may want to increase the timeout - # or grab the evaluation id and check the status later - timeout_seconds=600, # 10 minutes - ) - print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") - - # --- All results at once without pagination - results = await client.results.get_all(evaluation=evaluation) - print(f"Found {len(results)} results") - print(results) - -if __name__ == "__main__": - asyncio.run(main()) -``` - -This approach is much simpler than manual pagination as it: -- Automatically handles all pagination internally -- Returns all results in a single call -- No need to manage page numbers or loop through pages -- Works with both sync and async clients - -## Pagination - Get All Results (Manual) - -### Simple Approach - -```python -from atlas import Atlas - -client = Atlas() -evaluation_id = "your_evaluation_id" - -# Get all results across all pages - all_results = [] - page = 1 - - while True: - results_data = client.results.get_by_id( - evaluation_id=evaluation_id, - page=page, - page_size=100 # 100 results per page - ) - - if not results_data or not results_data.results: - break - - all_results.extend(results_data.results) - print(f"Loaded page {page}: {len(results_data.results)} results") - - # Check if we've reached the last page - if page >= results_data.pagination.total_pages: - break - - page += 1 - -print(f"\nTotal results collected: {len(all_results):,}") - -# Calculate accuracy -correct = sum(1 for r in all_results if r.score > 0.5) -accuracy = correct / len(all_results) if all_results else 0 -print(f"Overall accuracy: {accuracy:.1%} ({correct:,}/{len(all_results):,})") -``` - - -## Key Points - -- Results are paginated with default page size of 100 -- Always loop through all pages to get complete results -- Use `page_size` parameter to control how many results per page (1-500) -- Check `pagination.total_pages` to know when to stop -- Results contain `score`, `subset`, `prompt`, `result`, and `truth` fields -- Use async versions for better performance with large datasets diff --git a/docs/examples/timeouts.md b/docs/examples/timeouts.md deleted file mode 100644 index 6717512..0000000 --- a/docs/examples/timeouts.md +++ /dev/null @@ -1,231 +0,0 @@ -# Working with Timeouts - -Simple examples for configuring timeouts with the Atlas Python SDK. - -## Basic Timeouts - -### Set Client Timeout - -```python -from atlas import Atlas - -# Set timeout for all requests (in seconds) -client = Atlas(timeout=300.0) # 5 minutes - -# Use the client normally -models = client.models.get() -evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) -``` - -### Different Timeouts for Different Use Cases - -```python -from atlas import Atlas - -# Quick operations (testing, health checks) -quick_client = Atlas(timeout=30.0) # 30 seconds - -# Normal operations -normal_client = Atlas(timeout=300.0) # 5 minutes - -# Long operations (large evaluations) -patient_client = Atlas(timeout=1800.0) # 30 minutes - -# Use appropriate client -models = quick_client.models.get() # Fast operation -evaluation = patient_client.evaluations.create(...) # Slow operation -``` - -## Per-Request Timeouts - -### Override Timeout for Specific Requests - -```python -from atlas import Atlas - -client = Atlas(timeout=60.0) # Default 1 minute - -# Override timeout for specific operations -models = client.models.get() -benchmarks = client.benchmarks.get() - -# This evaluation might take longer, so use extended timeout -evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], - timeout=1800.0 # 30 minutes for this specific request -) -``` - -## Async Timeouts - -```python -import asyncio -from atlas import AsyncAtlas - -async def main(): - # Set timeout for async client - client = await AsyncAtlas.create(timeout=300.0) - - models = await client.models.get() - benchmarks = await client.benchmarks.get() - - # Create evaluation with custom timeout - evaluation = await client.evaluations.create( - model=models[0], - benchmark=benchmarks[0], - timeout=1200.0 # 20 minutes - ) - - return evaluation - -asyncio.run(main()) -``` - -## Handling Timeout Errors - -```python -from atlas import Atlas -import atlas - -def safe_operation_with_timeout(): - client = Atlas(timeout=60.0) # 1 minute timeout - - try: - models = client.models.get() - benchmarks = client.benchmarks.get() - - evaluation = client.evaluations.create( - model=models[0], - benchmark=benchmarks[0] - ) - - return evaluation - - except atlas.APITimeoutError: - print("Operation timed out") - print("Try increasing the timeout or check your connection") - return None - - except atlas.APIError as e: - print(f"Other API error: {e}") - return None - -result = safe_operation_with_timeout() -``` - -## Wait for Completion with Timeout - -```python -from atlas import Atlas - -client = Atlas() -models = client.models.get() -benchmarks = client.benchmarks.get() - -# Create evaluation -evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) - -# Wait for completion with timeout -try: - completed = client.evaluations.wait_for_completion( - evaluation, - interval_seconds=30, # Check every 30 seconds - timeout_seconds=3600 # Give up after 1 hour - ) - - if completed and completed.is_success: - print("Evaluation completed successfully") - else: - print("Evaluation failed") - -except atlas.APITimeoutError: - print("Evaluation did not complete within 1 hour") - print(f"Current status: {evaluation.status}") -``` - -## Advanced Timeout Configuration - -### Granular Control - -```python -import httpx -from atlas import Atlas - -# Configure different timeouts for different operations -client = Atlas( - timeout=httpx.Timeout( - connect=10.0, # 10 seconds to connect - read=600.0, # 10 minutes to read response - write=30.0, # 30 seconds to send request - pool=30.0 # 30 seconds for connection pool - ) -) - -# Use normally -evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) -``` - -## Recommended Timeouts - -### By Operation Type - -```python -from atlas import Atlas - -# Quick operations (< 30 seconds) -quick_client = Atlas(timeout=30.0) -models = quick_client.models.get() -benchmarks = quick_client.benchmarks.get() - -# Normal operations (5 minutes) -normal_client = Atlas(timeout=300.0) -evaluation = normal_client.evaluations.create(...) - -# Long operations (30+ minutes) -long_client = Atlas(timeout=1800.0) -completed = long_client.evaluations.wait_for_completion(...) - -# Getting results (depends on size) -results_client = Atlas(timeout=600.0) # 10 minutes -results = results_client.results.get_by_id(...) -``` - -## Production Best Practices - -### Adaptive Timeouts - -```python -from atlas import Atlas -import atlas - -def create_evaluation_with_adaptive_timeout(model, benchmark): - """Try with increasing timeouts""" - - timeouts = [60.0, 300.0, 900.0] # 1min, 5min, 15min - - for timeout in timeouts: - try: - client = Atlas(timeout=timeout) - - print(f"Trying with {timeout}s timeout...") - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - - print(f"Success with {timeout}s timeout") - return evaluation - - except atlas.APITimeoutError: - print(f"Timed out with {timeout}s timeout") - continue - except atlas.APIError as e: - print(f"API error (not timeout): {e}") - break - - print("Failed with all timeout attempts") - return None - -# Usage -models = Atlas().models.get() -benchmarks = Atlas().benchmarks.get() -evaluation = create_evaluation_with_adaptive_timeout(models[0], benchmarks[0]) -``` From f2b597e3ba1a3573b1904779e58e6064803f76ae Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Wed, 27 Aug 2025 09:27:34 -0400 Subject: [PATCH 13/15] simplify error code description and troubleshooting --- docs/troubleshooting/authentication.md | 155 +----- docs/troubleshooting/common-issues.md | 108 ---- docs/troubleshooting/error-codes.md | 708 +------------------------ 3 files changed, 4 insertions(+), 967 deletions(-) delete mode 100644 docs/troubleshooting/common-issues.md diff --git a/docs/troubleshooting/authentication.md b/docs/troubleshooting/authentication.md index 958ca27..007dd5b 100644 --- a/docs/troubleshooting/authentication.md +++ b/docs/troubleshooting/authentication.md @@ -1,14 +1,6 @@ # Authentication Problems -This guide covers authentication-related issues and their solutions when using the Atlas Python SDK. - -## Understanding Atlas Authentication - -The Atlas SDK uses API key-based authentication with three required components: - -1. **API Key**: Your secret authentication token -2. **Organization ID**: Your organization identifier -3. **Project ID**: The specific project you're working with +This guide covers common authentication-related issues when using the Layerlens python sdk. ## Common Authentication Errors @@ -24,147 +16,4 @@ The Atlas SDK uses API key-based authentication with three required components: ### Missing Required Configuration -**Error**: `AtlasError: The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable` - -**Solutions**: - -1. **Check all required environment variables**: - - ```bash - # Linux/macOS - echo $LAYERLENS_ATLAS_API_KEY - - # Windows - echo %LAYERLENS_ATLAS_API_KEY% - ``` - -2. **Set environment variables properly**: - - ```bash - # Linux/macOS - in your shell profile (.bashrc, .zshrc, etc.) - export LAYERLENS_ATLAS_API_KEY="sk-..." - - # Windows - persistently - setx LAYERLENS_ATLAS_API_KEY "sk-..." - ``` - -3. **Use .env file**: - - ```bash - # Create .env file in your project root - LAYERLENS_ATLAS_API_KEY=sk-your-key-here - ``` - - ```python - # Load .env file in your Python code - from dotenv import load_dotenv - import os - - load_dotenv() - - from atlas import Atlas - client = Atlas() - ``` - -### Permission Denied Errors - -**Error**: `PermissionDeniedError: 403 Forbidden` - -**Symptoms**: - -- Valid API key but still get 403 errors -- Can authenticate but cannot create evaluations -- Access denied to specific models or benchmarks - -**Diagnosis**: - -```python -import atlas -from atlas import Atlas - -def diagnose_permissions(): - client = Atlas() - - print("🔍 Permission Diagnosis:") - - # Test basic access - try: - # This should fail with specific error types - evaluation = client.evaluations.create( - model="test-model", - benchmark="test-benchmark" - ) - except atlas.AuthenticationError: - print(" ❌ Authentication failed - invalid API key") - return - except atlas.PermissionDeniedError: - print(" ❌ Permission denied - valid key, insufficient permissions") - except atlas.NotFoundError: - print(" ✅ Authentication works (model/benchmark not found is normal)") - except Exception as e: - print(f" ❓ Unexpected error: {e}") - - # Test with common models/benchmarks - test_combinations = [ - ("gpt-3.5-turbo", "mmlu"), - ("gpt-4", "hellaswag"), - ("claude-3-sonnet", "arc-challenge") - ] - - print("\n Testing access to specific resources:") - - for model, benchmark in test_combinations: - try: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - if evaluation: - print(f" ✅ {model} + {benchmark}: Access granted") - except atlas.PermissionDeniedError: - print(f" ❌ {model} + {benchmark}: Permission denied") - except atlas.NotFoundError: - print(f" ⚠️ {model} + {benchmark}: Resource not found") - except Exception as e: - print(f" ❓ {model} + {benchmark}: {e}") - -diagnose_permissions() -``` - -### Organization/Project Access Issues - -**Problem**: Valid API key but wrong organization or project - -**Symptoms**: - -- Authentication succeeds -- Cannot access expected models or benchmarks -- Permission errors for resources you should have access to - -**Diagnosis**: - -```python -import os -from atlas import Atlas -import atlas - -def verify_org_project_access(): - # Test with different org/project combinations - api_key = os.getenv('LAYERLENS_ATLAS_API_KEY') - - if not api_key: - print("❌ No API key found") - return - - try: - client = Atlas(api_key=api_key) - evaluation = client.evaluations.create(model="test", benchmark="test") - - except atlas.AuthenticationError: - print(" ❌ Authentication failed") - except atlas.PermissionDeniedError: - print(" ❌ Permission denied - check org/project IDs") - except atlas.NotFoundError: - print(" ✅ Access granted (test model not found is expected)") - except Exception as e: - print(f" ❓ Error: {e}") - -verify_org_project_access() -``` +**Error**: `AtlasError: The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable` \ No newline at end of file diff --git a/docs/troubleshooting/common-issues.md b/docs/troubleshooting/common-issues.md deleted file mode 100644 index 558a315..0000000 --- a/docs/troubleshooting/common-issues.md +++ /dev/null @@ -1,108 +0,0 @@ -# Common Issues - -This guide covers the most frequently encountered issues when using the Atlas Python SDK and provides step-by-step solutions. - -## Installation Issues - -### Package Not Found - -**Problem**: `pip install atlas` fails with "No matching distribution found" - -**Solutions**: - -1. **Check Python version compatibility**: - - ```bash - python --version - # Atlas requires Python 3.8+ - ``` - -2. **Update pip and try again**: - - ```bash - python -m pip install --upgrade pip - pip install atlas - ``` - -3. **Use Python 3 explicitly**: - ```bash - python3 -m pip install atlas - ``` - -## Configuration Issues - -### Missing Environment Variables - -**Problem**: `AtlasError: The api_key client option must be set` - -**Diagnosis**: - -```python -import os -print(f"API Key: {os.getenv('LAYERLENS_ATLAS_API_KEY', 'NOT SET')}") -``` - -**Solutions**: - -1. **Set environment variables**: - - ```bash - # Linux/macOS - export LAYERLENS_ATLAS_API_KEY="your_api_key_here" - - # Windows - set LAYERLENS_ATLAS_API_KEY=your_api_key_here - ``` - -2. **Use .env file**: - - ```bash - # Create .env file - LAYERLENS_ATLAS_API_KEY=your_api_key_here - ``` - - ```python - from dotenv import load_dotenv - load_dotenv() - - from atlas import Atlas - client = Atlas() - ``` - -3. **Pass explicitly to client**: - - ```python - from atlas import Atlas - - client = Atlas(api_key="your_api_key_here") - ``` - -### Where to Get Help - -1. **LayerLens Support**: Contact support through your LayerLens dashboard for technical issues -2. **Documentation**: Check the [complete documentation](../README.md) -3. **Community**: Join LayerLens community channels for discussions - -### Creating a Good Bug Report - -Include this information when reporting issues: - -1. **Environment details** (from debug info above) -2. **Complete error message** with stack trace -3. **Minimal reproducible example**: - - ```python - from atlas import Atlas - - client = Atlas() - - # Minimal code that demonstrates the problem - evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" - ) - ``` - -4. **Expected vs actual behavior** -5. **Steps to reproduce** -6. **Workarounds attempted** diff --git a/docs/troubleshooting/error-codes.md b/docs/troubleshooting/error-codes.md index fc27dc0..38a0a2e 100644 --- a/docs/troubleshooting/error-codes.md +++ b/docs/troubleshooting/error-codes.md @@ -1,6 +1,6 @@ # Error Codes Reference -This reference guide provides detailed information about all error codes and exceptions in the Atlas Python SDK. +This reference guide provides detailed information about all error codes and exceptions in the Layerlens Python SDK. ## Exception Hierarchy @@ -19,708 +19,4 @@ AtlasError (Base exception) │ ├── UnprocessableEntityError (422) │ ├── RateLimitError (429) │ └── InternalServerError (500+) -``` - -## HTTP Status Code Errors - -### 400 - Bad Request (`BadRequestError`) - -**When it occurs**: - -- Invalid request parameters -- Missing required fields -- Malformed request data - -**Common causes**: - -```python -# Empty or invalid parameters -client.evaluations.create(model="", benchmark="") # Empty strings -client.evaluations.create(model=None, benchmark="mmlu") # None values - -# Invalid parameter types -client.evaluations.create(model=123, benchmark="mmlu") # Wrong type -``` - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - evaluation = client.evaluations.create(model="", benchmark="mmlu") -except atlas.BadRequestError as e: - print(f"Bad request: {e}") - print(f"Status code: {e.status_code}") # 400 - print(f"Response body: {e.body}") -``` - -**Solutions**: - -1. **Validate parameters before making requests**: - - ```python - def validate_evaluation_params(model, benchmark): - if not model or not isinstance(model, str): - raise ValueError("Model must be a non-empty string") - if not benchmark or not isinstance(benchmark, str): - raise ValueError("Benchmark must be a non-empty string") - return True - - if validate_evaluation_params(model, benchmark): - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - ``` - -2. **Check parameter format requirements**: - - ```python - # Ensure parameters meet expected format - model = model.strip() if model else "" - benchmark = benchmark.strip() if benchmark else "" - - if len(model) < 2 or len(benchmark) < 2: - raise ValueError("Model and benchmark names must be at least 2 characters") - ``` - -### 401 - Unauthorized (`AuthenticationError`) - -**When it occurs**: - -- Missing API key -- Invalid or expired API key -- API key format issues - -**Common causes**: - -```python -# Missing API key -client = Atlas(api_key=None) - -# Invalid API key format -client = Atlas(api_key="invalid-key") - -# Expired API key (need to regenerate) -client = Atlas(api_key="sk-old-expired-key") -``` - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas(api_key="invalid-key") - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.AuthenticationError as e: - print(f"Authentication failed: {e}") - print(f"Status code: {e.status_code}") # 401 - print(f"Request ID: {e.request_id}") -``` - -**Solutions**: - -1. **Verify API key configuration**: - - ```python - import os - - api_key = os.getenv('LAYERLENS_ATLAS_API_KEY') - if not api_key: - print("❌ API key not found in environment variables") - elif len(api_key) < 10: - print("⚠️ API key seems too short") - else: - print("✅ API key found and looks valid") - ``` - -2. **Regenerate API key**: - - - Log into Atlas dashboard - - Go to Settings > API Keys - - Generate new API key - - Update environment variables - -3. **Test authentication separately**: - - ```python - def test_authentication(api_key): - try: - client = Atlas(api_key=api_key) - # Try minimal operation to test auth - client.evaluations.create(model="test", benchmark="test") - except atlas.AuthenticationError: - return False, "Invalid API key" - except atlas.NotFoundError: - return True, "Authentication successful (test resources not found is expected)" - except Exception as e: - return False, f"Unexpected error: {e}" - - is_valid, message = test_authentication(your_api_key) - print(f"Authentication test: {message}") - ``` - -### 403 - Forbidden (`PermissionDeniedError`) - -**When it occurs**: - -- Valid API key but insufficient permissions -- No access to specific models or benchmarks -- Organization/project access issues - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - evaluation = client.evaluations.create(model="restricted-model", benchmark="mmlu") -except atlas.PermissionDeniedError as e: - print(f"Permission denied: {e}") - print(f"Status code: {e.status_code}") # 403 - print(f"Response body: {e.body}") -``` - -**Solutions**: - -1. **Test access to different resources**: - - ```python - def test_resource_access(models, benchmarks): - client = Atlas() - access_matrix = {} - - for model in models: - access_matrix[model] = {} - for benchmark in benchmarks: - try: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - access_matrix[model][benchmark] = "✅ Access granted" - except atlas.PermissionDeniedError: - access_matrix[model][benchmark] = "❌ Permission denied" - except atlas.NotFoundError: - access_matrix[model][benchmark] = "❓ Resource not found" - except Exception as e: - access_matrix[model][benchmark] = f"❓ {type(e).__name__}" - - return access_matrix - - # Test common resources - models = ["gpt-3.5-turbo", "gpt-4", "claude-3-sonnet"] - benchmarks = ["mmlu", "hellaswag", "arc-easy"] - - access = test_resource_access(models, benchmarks) - ``` - -2. **Contact administrator for access**: - - Request access to specific models or benchmarks - - Verify project membership - - Check organization-level permissions - -### 404 - Not Found (`NotFoundError`) - -**When it occurs**: - -- Model ID doesn't exist -- Benchmark ID doesn't exist -- Evaluation ID not found (for results) -- Resource doesn't exist in your organization - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - evaluation = client.evaluations.create(model="nonexistent-model", benchmark="mmlu") -except atlas.NotFoundError as e: - print(f"Resource not found: {e}") - print(f"Status code: {e.status_code}") # 404 -``` - -**Solutions**: - -1. **Verify resource names**: - - ```python - def find_available_models(): - """Try common model names to find available ones""" - client = Atlas() - - common_models = [ - "gpt-4", "gpt-3.5-turbo", "gpt-4-turbo", - "claude-3-opus", "claude-3-sonnet", "claude-3-haiku", - "llama-2-70b", "llama-2-13b", "mistral-7b" - ] - - available_models = [] - - for model in common_models: - try: - # Test with common benchmark - evaluation = client.evaluations.create(model=model, benchmark="mmlu") - if evaluation: - available_models.append(model) - except atlas.NotFoundError: - # Model or benchmark not found - continue - except atlas.PermissionDeniedError: - # Model exists but no permission - available_models.append(f"{model} (no permission)") - except Exception: - # Other errors - model might exist - available_models.append(f"{model} (unknown status)") - - return available_models - - available = find_available_models() - print(f"Available models: {available}") - ``` - -2. **Check spelling and case sensitivity**: - - ```python - # Common mistakes - correct_names = { - "GPT-4": "gpt-4", - "GPT4": "gpt-4", - "MMLU": "mmlu", - "HellaSwag": "hellaswag", - "arc_challenge": "arc-challenge" # Underscore vs hyphen - } - ``` - -3. **Use exact names from Atlas dashboard**: - - Log into Atlas dashboard - - Check available models and benchmarks - - Copy exact names (case-sensitive) - -### 409 - Conflict (`ConflictError`) - -**When it occurs**: - -- Resource already exists -- Conflicting operation in progress -- State conflict (e.g., trying to modify completed evaluation) - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - # Some operation that conflicts with current state - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.ConflictError as e: - print(f"Conflict error: {e}") - print(f"Status code: {e.status_code}") # 409 -``` - -**Solutions**: - -1. **Check current resource state** -2. **Wait for ongoing operations to complete** -3. **Use different resource identifiers** - -### 422 - Unprocessable Entity (`UnprocessableEntityError`) - -**When it occurs**: - -- Valid request format but business logic prevents processing -- Parameter combinations that don't make sense -- Resource constraints exceeded - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - evaluation = client.evaluations.create(model="gpt-4", benchmark="invalid-benchmark") -except atlas.UnprocessableEntityError as e: - print(f"Unprocessable entity: {e}") - print(f"Status code: {e.status_code}") # 422 - print(f"Response details: {e.body}") -``` - -**Solutions**: - -1. **Check business logic constraints** -2. **Verify parameter combinations are valid** -3. **Review API documentation for limitations** - -### 429 - Rate Limited (`RateLimitError`) - -**When it occurs**: - -- Too many requests in short time period -- API rate limits exceeded -- Organization-level quotas reached - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - - # Making too many requests quickly - for i in range(100): - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") - -except atlas.RateLimitError as e: - print(f"Rate limited: {e}") - print(f"Status code: {e.status_code}") # 429 - print(f"Retry after: {e.response.headers.get('retry-after', 'not specified')}") -``` - -**Solutions**: - -1. **Implement retry with backoff**: - - ```python - import time - import atlas - from atlas import Atlas - - def create_evaluation_with_rate_limit_handling(model, benchmark, max_retries=3): - client = Atlas() - - for attempt in range(max_retries): - try: - return client.evaluations.create(model=model, benchmark=benchmark) - - except atlas.RateLimitError as e: - retry_after = e.response.headers.get('retry-after') - - if retry_after: - wait_time = int(retry_after) - print(f"Rate limited. Waiting {wait_time}s as requested...") - else: - wait_time = (2 ** attempt) * 60 # Exponential backoff - print(f"Rate limited. Waiting {wait_time}s...") - - if attempt < max_retries - 1: - time.sleep(wait_time) - else: - raise # Re-raise on final attempt - - return None - - evaluation = create_evaluation_with_rate_limit_handling("gpt-4", "mmlu") - ``` - -2. **Add delays between requests**: - - ```python - import time - - evaluations = [] - models = ["gpt-4", "claude-3-opus", "llama-2-70b"] - - for model in models: - evaluation = client.evaluations.create(model=model, benchmark="mmlu") - evaluations.append(evaluation) - - # Wait between requests to avoid rate limits - time.sleep(2) # 2-second delay - ``` - -3. **Monitor rate limit headers**: - ```python - def monitor_rate_limits(client): - """Monitor rate limit status""" - # This would require SDK modification to expose headers - # Check with LayerLens documentation for rate limit details - pass - ``` - -### 500+ - Server Errors (`InternalServerError`) - -**When it occurs**: - -- Atlas API server errors -- Temporary service unavailability -- Infrastructure issues - -**Example error**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas() - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.InternalServerError as e: - print(f"Server error: {e}") - print(f"Status code: {e.status_code}") # 500, 502, 503, etc. - print(f"Request ID: {e.request_id}") # Include in support requests -``` - -**Solutions**: - -1. **Implement retry logic**: - - ```python - import time - import atlas - from atlas import Atlas - - def create_evaluation_with_server_error_handling(model, benchmark): - client = Atlas() - max_retries = 3 - base_delay = 5 # seconds - - for attempt in range(max_retries): - try: - return client.evaluations.create(model=model, benchmark=benchmark) - - except atlas.InternalServerError as e: - print(f"Server error on attempt {attempt + 1}: {e}") - - if attempt < max_retries - 1: - # Exponential backoff with jitter - delay = base_delay * (2 ** attempt) + random.uniform(0, 2) - print(f"Retrying in {delay:.1f}s...") - time.sleep(delay) - else: - print(f"All {max_retries} attempts failed. Request ID: {e.request_id}") - raise - - return None - ``` - -2. **Check service status**: - - - Visit LayerLens status page - - Check for ongoing incidents - - Monitor Atlas service announcements - -3. **Report persistent issues**: - - Include request ID from error - - Provide timestamp and error details - - Contact LayerLens support - -## Connection Errors - -### `APIConnectionError` - -**When it occurs**: - -- Network connectivity issues -- DNS resolution failures -- Firewall blocking requests -- Proxy configuration problems - -**Example**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas(timeout=10.0) - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.APIConnectionError as e: - print(f"Connection error: {e}") - print(f"Request URL: {e.request.url}") -``` - -**Solutions**: - -1. **Test basic connectivity**: - - ```bash - ping api.layerlens.com - curl -I https://api.layerlens.com - ``` - -2. **Check proxy/firewall settings** -3. **Verify DNS resolution** - -### `APITimeoutError` - -**When it occurs**: - -- Request takes longer than configured timeout -- Network latency issues -- Server processing delays - -**Example**: - -```python -import atlas -from atlas import Atlas - -try: - client = Atlas(timeout=30.0) # 30-second timeout - evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except atlas.APITimeoutError as e: - print(f"Request timed out: {e}") -``` - -**Solutions**: - -1. **Increase timeout**: - - ```python - client = Atlas(timeout=600.0) # 10 minutes - ``` - -2. **Use appropriate timeouts for operation type**: - - ```python - # Quick operations - quick_client = Atlas(timeout=60.0) - - # Long-running evaluations - patient_client = Atlas(timeout=1800.0) # 30 minutes - ``` - -## Error Handling Best Practices - -### Comprehensive Error Handling - -```python -import atlas -from atlas import Atlas -import time -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def robust_create_evaluation(model: str, benchmark: str): - """Create evaluation with comprehensive error handling""" - client = Atlas() - - try: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - - if evaluation: - logger.info(f"✅ Evaluation created: {evaluation.id}") - return evaluation - else: - logger.warning("⚠️ Evaluation creation returned None") - return None - - except atlas.BadRequestError as e: - logger.error(f"❌ Bad request - check parameters: {e}") - logger.error(f" Model: '{model}', Benchmark: '{benchmark}'") - return None - - except atlas.AuthenticationError as e: - logger.error(f"❌ Authentication failed: {e}") - logger.error(" Check API key configuration") - return None - - except atlas.PermissionDeniedError as e: - logger.error(f"❌ Permission denied: {e}") - logger.error(f" No access to model '{model}' or benchmark '{benchmark}'") - return None - - except atlas.NotFoundError as e: - logger.error(f"❌ Resource not found: {e}") - logger.error(f" Model '{model}' or benchmark '{benchmark}' doesn't exist") - return None - - except atlas.RateLimitError as e: - retry_after = e.response.headers.get('retry-after', 60) - logger.warning(f"⏳ Rate limited - retry after {retry_after}s") - return None # Could implement retry logic here - - except atlas.InternalServerError as e: - logger.error(f"❌ Server error: {e}") - logger.error(f" Request ID: {e.request_id} (include in support requests)") - return None - - except atlas.APITimeoutError as e: - logger.error(f"⏰ Request timed out: {e}") - logger.error(" Consider increasing timeout or checking network") - return None - - except atlas.APIConnectionError as e: - logger.error(f"🔌 Connection error: {e}") - logger.error(" Check network connectivity and proxy settings") - return None - - except atlas.APIError as e: - logger.error(f"❌ Unexpected API error: {e}") - logger.error(f" Type: {type(e).__name__}") - return None - - except Exception as e: - logger.error(f"❌ Unexpected error: {e}") - logger.error(f" Type: {type(e).__name__}") - return None - -# Usage -evaluation = robust_create_evaluation("gpt-4", "mmlu") -``` - -### Error Recovery Patterns - -```python -import atlas -from atlas import Atlas -import time -import random - -class AtlasErrorRecovery: - """Implement various error recovery patterns""" - - def __init__(self, client: Atlas): - self.client = client - - def exponential_backoff_retry(self, operation, max_retries=3, base_delay=1): - """Retry with exponential backoff""" - for attempt in range(max_retries): - try: - return operation() - except (atlas.InternalServerError, atlas.APIConnectionError, atlas.APITimeoutError) as e: - if attempt == max_retries - 1: - raise # Last attempt - re-raise the error - - delay = base_delay * (2 ** attempt) + random.uniform(0, 1) - print(f"Attempt {attempt + 1} failed: {e}") - print(f"Retrying in {delay:.1f}s...") - time.sleep(delay) - - def circuit_breaker(self, operation, failure_threshold=5, recovery_time=60): - """Implement circuit breaker pattern""" - # This would be a more complex implementation - # See advanced-usage.md for full implementation - pass - - def fallback_strategy(self, primary_operation, fallback_operation): - """Try primary operation, fall back to alternative""" - try: - return primary_operation() - except atlas.APIError as e: - print(f"Primary operation failed: {e}") - print("Trying fallback...") - return fallback_operation() - -# Usage -client = Atlas() -recovery = AtlasErrorRecovery(client) - -def create_evaluation(): - return client.evaluations.create(model="gpt-4", benchmark="mmlu") - -# Retry with exponential backoff -evaluation = recovery.exponential_backoff_retry(create_evaluation) -``` +``` \ No newline at end of file From 465466c66f8646743b33333dbdeec09fbaaa2c6d Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Wed, 27 Aug 2025 09:31:31 -0400 Subject: [PATCH 14/15] change name of section --- SUMMARY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SUMMARY.md b/SUMMARY.md index 17af81c..d7ebbf5 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -1,5 +1,5 @@ # Table of contents -* [Layerlens Python SDK Documentation](docs/README.md) +* [Layerlens Python SDK](docs/README.md) * [Api-reference](docs/api-reference/README.md) * [Client Configuration](docs/api-reference/client.md) * [Models & Benchmarks](docs/api-reference/models-benchmarks.md) From 4885a53683acbbd4413e34a9bf8e0ea7821e16c3 Mon Sep 17 00:00:00 2001 From: Robert Leonard Date: Wed, 27 Aug 2025 09:54:31 -0400 Subject: [PATCH 15/15] fix linting --- examples/async_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/async_client.py b/examples/async_client.py index 579084a..571e8cc 100644 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -10,7 +10,7 @@ async def main(): client = AsyncAtlas() # --- Models - models = await client.models.get(type="public",name="gpt-4o") + models = await client.models.get(type="public", name="gpt-4o") print(f"Models found: {models}") if not models: