diff --git a/README.md b/README.md index a6de2ce..2b9871a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # LayerLens Stratix Python SDK -The official Python library for the [LayerLens Stratix](https://layerlens.ai) evaluation API. +The official Python library for the [LayerLens Stratix](https://app.layerlens.ai) evaluation API. [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) diff --git a/docs/README.md b/docs/README.md index a6de2ce..2b9871a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # LayerLens Stratix Python SDK -The official Python library for the [LayerLens Stratix](https://layerlens.ai) evaluation API. +The official Python library for the [LayerLens Stratix](https://app.layerlens.ai) evaluation API. [![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) diff --git a/docs/getting-started/authentication.md b/docs/getting-started/authentication.md new file mode 100644 index 0000000..eb79f5f --- /dev/null +++ b/docs/getting-started/authentication.md @@ -0,0 +1,79 @@ +# Authentication & Configuration + +## API Key Setup + +The SDK authenticates using an API key tied to your LayerLens organization. You can obtain your key from the [LayerLens dashboard](https://app.layerlens.ai). + +### Environment Variable (Recommended) + +Set the `LAYERLENS_STRATIX_API_KEY` environment variable and the client will pick it up automatically: + +```bash +export LAYERLENS_STRATIX_API_KEY="your-api-key" +``` + +```python +from layerlens import Stratix + +# Automatically reads from LAYERLENS_STRATIX_API_KEY +client = Stratix() +``` + +### Explicit API Key + +Pass the key directly when constructing the client: + +```python +from layerlens import Stratix + +client = Stratix(api_key="your-api-key") +``` + +### Using a .env File + +```bash +# .env (add this file to .gitignore) +LAYERLENS_STRATIX_API_KEY=your-api-key +``` + +```python +from dotenv import load_dotenv +load_dotenv() + +from layerlens import Stratix +client = Stratix() +``` + +## Configuration Options + +| Parameter | Type | Default | Description | +| ---------- | -------------------------------- | --------------------------------- | --------------- | +| `api_key` | `str \| None` | `LAYERLENS_STRATIX_API_KEY` env | Your API key | +| `base_url` | `str \| httpx.URL \| None` | `https://api.layerlens.ai/api/v1` | API base URL | +| `timeout` | `float \| httpx.Timeout \| None` | 10 minutes | Request timeout | + +## Environment Variables + +| Variable | Description | Default | +| ---------------------------- | ------------------------- | --------------------------------- | +| `LAYERLENS_STRATIX_API_KEY` | Your API key | (required) | +| `LAYERLENS_STRATIX_BASE_URL` | Override the API base URL | `https://api.layerlens.ai/api/v1` | + +Legacy environment variables (`LAYERLENS_ATLAS_API_KEY`, `LAYERLENS_ATLAS_BASE_URL`) are also supported for backward compatibility. + +## Organization & Project + +On initialization, the client fetches your organization and project IDs automatically using your API key. These are used to scope all API requests. + +## Timeout Configuration + +```python +# Global timeout (30 seconds) +client = Stratix(timeout=30.0) + +# Per-request timeout override +evaluation = client.with_options(timeout=120.0).evaluations.create( + model=model, + benchmark=benchmark, +) +``` diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..1d26b71 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,47 @@ +# Installation + +## Requirements + +- Python 3.8 or higher +- pip package manager + +## Install from LayerLens Package Registry + +```bash +pip install layerlens --extra-index-url https://sdk.layerlens.ai/package +``` + +## Verify Installation + +```python +import layerlens +print(layerlens.__version__) +``` + +## Dependencies + +The SDK installs the following dependencies automatically: + +- `httpx` - HTTP client for API requests +- `pydantic` - Data validation and serialization +- `requests` - Used for file uploads (presigned S3 URLs) + +## Upgrading + +To upgrade to the latest version: + +```bash +pip install --upgrade layerlens --extra-index-url https://sdk.layerlens.ai/package +``` + +## Virtual Environment (Recommended) + +It's recommended to install the SDK in a virtual environment to avoid dependency conflicts: + +```bash +python -m venv .venv +source .venv/bin/activate # Linux/macOS +# .venv\Scripts\activate # Windows + +pip install layerlens --extra-index-url https://sdk.layerlens.ai/package +``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..f9f32b4 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,130 @@ +# Quick Start Guide + +This guide walks you through the most common SDK workflows. + +## Setup + +```bash +pip install layerlens --extra-index-url https://sdk.layerlens.ai/package +export LAYERLENS_STRATIX_API_KEY="your-api-key" +``` + +## Run a Benchmark Evaluation + +```python +from layerlens import Stratix + +client = Stratix() + +# Get a model and benchmark by key +model = client.models.get_by_key("openai/gpt-4o") +benchmark = client.benchmarks.get_by_key("arc-agi-2") + +# Create an evaluation +evaluation = client.evaluations.create( + model=model, + benchmark=benchmark, +) + +# Wait for results +result = client.evaluations.wait_for_completion(evaluation) +print(f"Accuracy: {result.accuracy}") +``` + +## Create a Judge and Evaluate Traces + +```python +import time +from layerlens import Stratix + +client = Stratix() + +# Create a judge +judge = client.judges.create( + name="Response Quality Judge", + evaluation_goal="Rate whether the response is accurate, complete, and well-structured", +) + +# Upload traces from a JSON/JSONL file +upload = client.traces.upload("./my_traces.json") +print(f"Uploaded {len(upload.trace_ids)} traces") + +# Run a trace evaluation +trace_eval = client.trace_evaluations.create( + trace_id=upload.trace_ids[0], + judge_id=judge.id, +) + +# Poll until complete +while True: + evaluation = client.trace_evaluations.get(trace_eval.id) + if evaluation.status.value in ("success", "failure"): + break + time.sleep(2) + +# Get results +result = client.trace_evaluations.get_results(trace_eval.id) +if result: + print(f"Score: {result.score}, Passed: {result.passed}") + print(f"Reasoning: {result.reasoning}") +``` + +## Async Usage + +Every method is available in async form via `AsyncStratix`: + +```python +import asyncio +from layerlens import AsyncStratix + +async def main(): + client = AsyncStratix() + + model = await client.models.get_by_key("openai/gpt-4o") + benchmark = await client.benchmarks.get_by_key("arc-agi-2") + + evaluation = await client.evaluations.create( + model=model, + benchmark=benchmark, + ) + + result = await client.evaluations.wait_for_completion(evaluation) + print(f"Accuracy: {result.accuracy}") + +asyncio.run(main()) +``` + +## Browse Public Data + +```python +from layerlens import Stratix + +client = Stratix() + +# List public models +models = client.public.models.get() +for model in models.models: + print(f"{model.key}: {model.name}") + +# List public benchmarks +benchmarks = client.public.benchmarks.get() +for bm in benchmarks.benchmarks: + print(f"{bm.key}: {bm.name}") +``` + +## Error Handling + +```python +from layerlens import Stratix, NotFoundError, AuthenticationError, APIError + +client = Stratix() + +try: + model = client.models.get_by_id("nonexistent-id") +except NotFoundError as e: + print(f"Not found: {e.message}") +except AuthenticationError as e: + print(f"Auth failed: {e.message}") +except APIError as e: + print(f"API error ({e.status_code}): {e.message}") +``` diff --git a/docs/security/data-privacy.md b/docs/security/data-privacy.md new file mode 100644 index 0000000..de4577f --- /dev/null +++ b/docs/security/data-privacy.md @@ -0,0 +1,51 @@ +# Data Privacy + +This guide covers data privacy considerations when using the LayerLens Stratix SDK. + +## Data Sent to the API + +When using the SDK, the following data is transmitted to LayerLens servers: + +- **API key** - Used for authentication (sent in request headers) +- **Trace data** - Contents of trace files you upload via `client.traces.upload()` +- **Evaluation parameters** - Model and benchmark selections, judge configurations +- **Judge definitions** - Evaluation goals and settings for custom judges + +## Data Storage + +- Uploaded traces are stored in your organization's project scope and are not shared across organizations +- Evaluation results are associated with your organization and project +- API keys are never logged or stored in SDK-side logs + +## Sensitive Data in Traces + +If your traces contain sensitive information (PII, credentials, proprietary data), consider: + +1. **Sanitize before upload** - Remove or redact sensitive fields from trace files before uploading +2. **Use test data** - Use synthetic or anonymized data for development and testing +3. **Review trace contents** - Inspect trace files before upload to ensure no secrets are included + +## Logging + +The SDK uses Python's standard `logging` module. Sensitive headers (API keys, authorization tokens) are automatically redacted in log output. + +```python +import logging + +# Enable SDK logging (API keys will be redacted) +logging.basicConfig(level=logging.DEBUG) +``` + +## Local Data + +The SDK does not write any data to disk. All operations are performed in-memory and transmitted directly to the API. Trace files are read from disk for upload but are not modified or cached. + +## Environment Variable Security + +Store API keys in environment variables rather than in source code: + +```bash +export LAYERLENS_STRATIX_API_KEY="your-api-key" +``` + +See [API Key Management](api-key-management.md) and [Environment Variables](environment-variables.md) for detailed guidance. diff --git a/docs/troubleshooting/common-issues.md b/docs/troubleshooting/common-issues.md new file mode 100644 index 0000000..3f286c2 --- /dev/null +++ b/docs/troubleshooting/common-issues.md @@ -0,0 +1,110 @@ +# Common Issues + +This guide covers frequently encountered issues when using the LayerLens Python SDK and how to resolve them. + +## Installation Issues + +### Package Not Found + +**Error**: `Could not find a version that satisfies the requirement layerlens` + +Make sure to include the custom package index: + +```bash +pip install layerlens --extra-index-url https://sdk.layerlens.ai/package +``` + +### Python Version Incompatibility + +**Error**: `Requires-Python >=3.8` + +The SDK requires Python 3.8 or higher. Check your version: + +```bash +python --version +``` + +## Client Initialization + +### Missing API Key + +**Error**: `StratixError: The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_STRATIX_API_KEY environment variable` + +Set the environment variable or pass the key explicitly: + +```bash +export LAYERLENS_STRATIX_API_KEY="your-api-key" +``` + +```python +client = Stratix(api_key="your-api-key") +``` + +### Organization Not Found + +**Error**: `NotFoundError` during client initialization + +Your API key may be invalid or expired. Verify it in the [LayerLens dashboard](https://app.layerlens.ai). + +## Evaluation Issues + +### Evaluation Stuck in Progress + +If `wait_for_completion` seems to hang, the evaluation may be queued behind other jobs. You can check the status manually: + +```python +evaluation = client.evaluations.get(evaluation.id) +print(f"Status: {evaluation.status}") +``` + +### Model or Benchmark Not Found + +When using `get_by_key`, ensure the key matches exactly (case-sensitive): + +```python +# Correct +model = client.models.get_by_key("openai/gpt-4o") + +# Search by name if unsure +models = client.models.get(type="public", name="gpt-4o") +``` + +## Trace Upload Issues + +### File Too Large + +Trace files must be under 50 MB. For larger datasets, split the file into smaller chunks. + +### Invalid File Format + +Trace uploads accept `.json` and `.jsonl` files only. Ensure your file contains valid JSON. + +## Timeout Errors + +**Error**: `APITimeoutError` + +The default timeout is 10 minutes. For long-running operations, increase it: + +```python +# Global timeout +client = Stratix(timeout=1200.0) # 20 minutes + +# Per-request timeout +result = client.with_options(timeout=300.0).evaluations.create( + model=model, + benchmark=benchmark, +) +``` + +## Network Issues + +**Error**: `APIConnectionError` + +- Verify your internet connection +- Check if `https://api.layerlens.ai` is reachable +- If behind a proxy, configure `httpx` accordingly + +## See Also + +- [Authentication Problems](authentication.md) for auth-specific issues +- [Error Codes Reference](error-codes.md) for the full exception hierarchy diff --git a/examples/async_judges_and_traces.py b/examples/async_judges_and_traces.py index 90ca657..1822d8d 100644 --- a/examples/async_judges_and_traces.py +++ b/examples/async_judges_and_traces.py @@ -64,10 +64,9 @@ async def main(): if not evaluation: continue try: - results_response = await client.trace_evaluations.get_results(evaluation.id) - if results_response and results_response.results: - for result in results_response.results: - print(f" Score: {result.score}, Passed: {result.passed}") + result = await client.trace_evaluations.get_results(evaluation.id) + if result: + print(f" Score: {result.score}, Passed: {result.passed}") else: print(f" Evaluation {evaluation.id}: no results yet") except Exception: diff --git a/examples/trace_evaluations.py b/examples/trace_evaluations.py index 5100d72..0891f19 100644 --- a/examples/trace_evaluations.py +++ b/examples/trace_evaluations.py @@ -58,14 +58,13 @@ # --- Get evaluation results (may 404 if still in progress) try: - results_response = client.trace_evaluations.get_results(evaluation.id) - if results_response and results_response.results: - for result in results_response.results: - print(f" Score: {result.score}, Passed: {result.passed}") - print(f" Reasoning: {result.reasoning}") - if result.steps: - for step in result.steps: - print(f" Step {step.step}: {step.reasoning}") + result = client.trace_evaluations.get_results(evaluation.id) + if result: + print(f" Score: {result.score}, Passed: {result.passed}") + print(f" Reasoning: {result.reasoning}") + if result.steps: + for step in result.steps: + print(f" Tool: {step.tool}, Result: {step.result[:80]}") else: print(" No results returned") except Exception: diff --git a/src/layerlens/models/api.py b/src/layerlens/models/api.py index bb36468..72a2390 100644 --- a/src/layerlens/models/api.py +++ b/src/layerlens/models/api.py @@ -117,8 +117,8 @@ class TraceEvaluationsResponse(BaseModel): total: int -class TraceEvaluationResultsResponse(BaseModel): - results: List[TraceEvaluationResult] +class TraceEvaluationResultsResponse(TraceEvaluationResult): + pass class CostEstimateResponse(BaseModel): diff --git a/src/layerlens/models/trace_evaluation.py b/src/layerlens/models/trace_evaluation.py index 054044c..98f32db 100644 --- a/src/layerlens/models/trace_evaluation.py +++ b/src/layerlens/models/trace_evaluation.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import Enum -from typing import List, Optional +from typing import Any, Dict, List, Optional from pydantic import Field, BaseModel, ConfigDict @@ -36,8 +36,9 @@ class TraceEvaluation(BaseModel): class TraceEvaluationStep(BaseModel): - step: int - reasoning: str + tool: str + args: Dict[str, Any] = {} + result: str = "" class TraceEvaluationResult(BaseModel): diff --git a/src/layerlens/resources/trace_evaluations/trace_evaluations.py b/src/layerlens/resources/trace_evaluations/trace_evaluations.py index 484f657..21fd2d2 100644 --- a/src/layerlens/resources/trace_evaluations/trace_evaluations.py +++ b/src/layerlens/resources/trace_evaluations/trace_evaluations.py @@ -7,7 +7,6 @@ from ...models import ( TraceEvaluation, CostEstimateResponse, - TraceEvaluationResult, TraceEvaluationsResponse, TraceEvaluationResultsResponse, ) @@ -146,12 +145,8 @@ def get_results( if not data or not isinstance(data, dict): return None - results = [ - r if isinstance(r, TraceEvaluationResult) else TraceEvaluationResult(**r) for r in data.get("results", []) - ] - try: - return TraceEvaluationResultsResponse(results=results) + return TraceEvaluationResultsResponse(**data) except Exception: return None @@ -297,12 +292,8 @@ async def get_results( if not data or not isinstance(data, dict): return None - results = [ - r if isinstance(r, TraceEvaluationResult) else TraceEvaluationResult(**r) for r in data.get("results", []) - ] - try: - return TraceEvaluationResultsResponse(results=results) + return TraceEvaluationResultsResponse(**data) except Exception: return None diff --git a/tests/resources/test_trace_evaluations.py b/tests/resources/test_trace_evaluations.py index b8e0295..c18ee09 100644 --- a/tests/resources/test_trace_evaluations.py +++ b/tests/resources/test_trace_evaluations.py @@ -66,8 +66,8 @@ def sample_result_data(self): "passed": True, "reasoning": "The output meets quality standards", "steps": [ - {"step": 1, "reasoning": "Checked correctness"}, - {"step": 2, "reasoning": "Checked style"}, + {"tool": "jq", "args": {"query": "."}, "result": "Checked correctness"}, + {"tool": "submit_evaluation", "args": {"score": 0.85}, "result": "Checked style"}, ], "model": "claude-sonnet-4-20250514", "turns": 3, @@ -232,26 +232,22 @@ def test_get_many_none_response(self, trace_evals_resource): def test_get_results_success(self, trace_evals_resource, sample_result_data): """get_results returns TraceEvaluationResultsResponse on success.""" - trace_evals_resource._get.return_value = { - "results": [sample_result_data], - } + trace_evals_resource._get.return_value = sample_result_data result = trace_evals_resource.get_results("te-123") assert isinstance(result, TraceEvaluationResultsResponse) - assert len(result.results) == 1 - res = result.results[0] - assert res.id == "result-123" - assert res.score == 0.85 - assert res.passed is True - assert res.reasoning == "The output meets quality standards" - assert len(res.steps) == 2 - assert res.model == "claude-sonnet-4-20250514" - assert res.latency_ms == 2500 + assert result.id == "result-123" + assert result.score == 0.85 + assert result.passed is True + assert result.reasoning == "The output meets quality standards" + assert len(result.steps) == 2 + assert result.model == "claude-sonnet-4-20250514" + assert result.latency_ms == 2500 def test_get_results_request_parameters(self, trace_evals_resource, sample_result_data): """get_results makes correct API request.""" - trace_evals_resource._get.return_value = {"results": [sample_result_data]} + trace_evals_resource._get.return_value = sample_result_data trace_evals_resource.get_results("te-123") @@ -262,13 +258,12 @@ def test_get_results_request_parameters(self, trace_evals_resource, sample_resul ) def test_get_results_empty(self, trace_evals_resource): - """get_results returns empty results list.""" - trace_evals_resource._get.return_value = {"results": []} + """get_results returns None when response has no data.""" + trace_evals_resource._get.return_value = {} result = trace_evals_resource.get_results("te-123") - assert isinstance(result, TraceEvaluationResultsResponse) - assert len(result.results) == 0 + assert result is None def test_get_results_none_response(self, trace_evals_resource): """get_results returns None when response is invalid."""