From 3dc2688e0e1e60797eda9e2daec7791cace1d294 Mon Sep 17 00:00:00 2001 From: m-peko Date: Thu, 11 Dec 2025 06:51:35 +0100 Subject: [PATCH 1/2] Add Apache 2.0 license --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4100e22 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2025 LayerLens + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From e28024c704e8f7c27e564db7f6bd88325220c7c9 Mon Sep 17 00:00:00 2001 From: Marin Peko <26385728+m-peko@users.noreply.github.com> Date: Tue, 24 Feb 2026 09:18:04 +0100 Subject: [PATCH 2/2] Rebranding and judges & traces support (#45) * Rename Atlas to Stratix * Integrate traces and judges * Support for judge optimizations * Fix example * Fix judges examples and implementation * Bump version * Add optimization example --- .github/workflows/publish-to-aws.yaml | 10 +- SUMMARY.md | 5 + docs/README.md | 20 +- docs/SUMMARY.md | 7 +- docs/api-reference/client.md | 32 +- docs/api-reference/errors.md | 98 ++-- docs/api-reference/evaluations.md | 40 +- docs/api-reference/judge-optimizations.md | 279 ++++++++++ docs/api-reference/judges.md | 213 ++++++++ docs/api-reference/models-benchmarks.md | 22 +- docs/api-reference/results.md | 26 +- docs/api-reference/trace-evaluations.md | 266 ++++++++++ docs/api-reference/traces.md | 229 +++++++++ docs/examples/creating-evaluations.md | 22 +- docs/examples/judges-and-traces.md | 275 ++++++++++ docs/security/api-key-management.md | 22 +- docs/security/environment-variables.md | 40 +- docs/security/rate-limiting.md | 6 +- docs/troubleshooting/authentication.md | 2 +- docs/troubleshooting/error-codes.md | 2 +- examples/all_results_no_pagination.py | 4 +- examples/async_client.py | 4 +- examples/async_client_simple.py | 4 +- examples/async_judges_and_traces.py | 82 +++ examples/async_run_evaluations.py | 4 +- examples/client.py | 4 +- examples/client_simple.py | 4 +- examples/fetch_results_async.py | 4 +- examples/get_benchmarks.py | 4 +- examples/get_evaluation.py | 6 +- examples/get_models.py | 4 +- examples/judge_optimizations.py | 113 ++++ examples/judges.py | 46 ++ examples/paginated_results.py | 4 +- examples/trace_evaluations.py | 80 +++ examples/traces.jsonl | 3 + examples/traces.py | 39 ++ package-lock.json | 2 +- pyproject.toml | 8 +- scripts/push_release_tag.sh | 4 +- src/CONTRIBUTING.md | 16 +- src/README.md | 22 +- src/layerlens/__init__.py | 53 +- src/layerlens/_base_client.py | 44 ++ src/layerlens/_client.py | 104 +++- src/layerlens/_exceptions.py | 8 +- src/layerlens/_resource.py | 14 +- src/layerlens/_version.py | 2 +- src/layerlens/models/__init__.py | 55 ++ src/layerlens/models/api.py | 85 ++++ src/layerlens/models/evaluation.py | 30 +- src/layerlens/models/judge.py | 32 ++ src/layerlens/models/judge_optimization.py | 46 ++ src/layerlens/models/trace.py | 29 ++ src/layerlens/models/trace_evaluation.py | 58 +++ .../resources/judge_optimizations/__init__.py | 3 + .../judge_optimizations.py | 306 +++++++++++ src/layerlens/resources/judges/__init__.py | 3 + src/layerlens/resources/judges/judges.py | 296 +++++++++++ .../resources/trace_evaluations/__init__.py | 3 + .../trace_evaluations/trace_evaluations.py | 328 ++++++++++++ src/layerlens/resources/traces/__init__.py | 3 + src/layerlens/resources/traces/traces.py | 385 ++++++++++++++ tests/conftest.py | 4 +- tests/resources/test_benchmarks.py | 6 +- tests/resources/test_evaluations.py | 4 +- tests/resources/test_judge_optimizations.py | 481 ++++++++++++++++++ tests/resources/test_judges.py | 448 ++++++++++++++++ tests/resources/test_models_resource.py | 6 +- tests/resources/test_results.py | 10 +- tests/resources/test_trace_evaluations.py | 444 ++++++++++++++++ tests/resources/test_traces.py | 436 ++++++++++++++++ tests/test_client.py | 120 ++--- tests/test_exceptions.py | 18 +- tests/test_integration.py | 102 ++-- tests/test_resource.py | 4 +- tests/test_utils.py | 4 +- 77 files changed, 5644 insertions(+), 407 deletions(-) create mode 100644 docs/api-reference/judge-optimizations.md create mode 100644 docs/api-reference/judges.md create mode 100644 docs/api-reference/trace-evaluations.md create mode 100644 docs/api-reference/traces.md create mode 100644 docs/examples/judges-and-traces.md create mode 100644 examples/async_judges_and_traces.py create mode 100644 examples/judge_optimizations.py create mode 100644 examples/judges.py create mode 100644 examples/trace_evaluations.py create mode 100644 examples/traces.jsonl create mode 100644 examples/traces.py create mode 100644 src/layerlens/models/judge.py create mode 100644 src/layerlens/models/judge_optimization.py create mode 100644 src/layerlens/models/trace.py create mode 100644 src/layerlens/models/trace_evaluation.py create mode 100644 src/layerlens/resources/judge_optimizations/__init__.py create mode 100644 src/layerlens/resources/judge_optimizations/judge_optimizations.py create mode 100644 src/layerlens/resources/judges/__init__.py create mode 100644 src/layerlens/resources/judges/judges.py create mode 100644 src/layerlens/resources/trace_evaluations/__init__.py create mode 100644 src/layerlens/resources/trace_evaluations/trace_evaluations.py create mode 100644 src/layerlens/resources/traces/__init__.py create mode 100644 src/layerlens/resources/traces/traces.py create mode 100644 tests/resources/test_judge_optimizations.py create mode 100644 tests/resources/test_judges.py create mode 100644 tests/resources/test_trace_evaluations.py create mode 100644 tests/resources/test_traces.py diff --git a/.github/workflows/publish-to-aws.yaml b/.github/workflows/publish-to-aws.yaml index beb6d00..6e1739c 100644 --- a/.github/workflows/publish-to-aws.yaml +++ b/.github/workflows/publish-to-aws.yaml @@ -63,12 +63,12 @@ jobs: ls -la echo "Checking if version file exists..." - if [ -f "src/atlas/_version.py" ]; then + if [ -f "src/layerlens/_version.py" ]; then echo "Version file found:" - cat src/atlas/_version.py + cat src/layerlens/_version.py # Extract version directly - VERSION=$(grep -E '^__version__\s*=' src/atlas/_version.py | grep -o '".*"' | tr -d '"') + VERSION=$(grep -E '^__version__\s*=' src/layerlens/_version.py | grep -o '".*"' | tr -d '"') echo "Direct extraction result: '$VERSION'" else echo "Version file not found, trying script method..." @@ -293,10 +293,10 @@ jobs: - LayerLens Atlas SDK + LayerLens Stratix SDK -

LayerLens Atlas SDK

+

LayerLens Stratix SDK

${{ env.PACKAGE_NAME }}
diff --git a/SUMMARY.md b/SUMMARY.md index d7ebbf5..4895dd7 100644 --- a/SUMMARY.md +++ b/SUMMARY.md @@ -4,9 +4,14 @@ * [Client Configuration](docs/api-reference/client.md) * [Models & Benchmarks](docs/api-reference/models-benchmarks.md) * [Evaluations](docs/api-reference/evaluations.md) + * [Judges](docs/api-reference/judges.md) + * [Traces](docs/api-reference/traces.md) + * [Trace Evaluations](docs/api-reference/trace-evaluations.md) + * [Judge Optimizations](docs/api-reference/judge-optimizations.md) * [Error Handling](docs/api-reference/errors.md) * [Examples](docs/examples/README.md) * [Running Evaluations](docs/examples/creating-evaluations.md) + * [Judges and Traces](docs/examples/judges-and-traces.md) * [Troubleshooting](docs/troubleshooting/README.md) * [Authentication Problems](docs/troubleshooting/authentication.md) * [Error Codes Reference](docs/troubleshooting/error-codes.md) diff --git a/docs/README.md b/docs/README.md index 9b2140e..d1dbfa4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,10 +1,10 @@ # Layerlens Python SDK Documentation -Welcome to the official documentation for the Layerlens Python SDK for the atlas platform. This library provides convenient programmatic to the Atlas platform from any Python 3.8+ application. +Welcome to the official documentation for the Layerlens Python SDK for the Stratix platform. This library provides convenient programmatic to the Stratix platform from any Python 3.8+ application. -## What is Atlas? +## What is Stratix? -Atlas is an evaluation platform that allows you to benchmark AI models against various datasets and metrics. The Python SDK provides two HTTP clients (syncronous and asynchronous) powered by [httpx](https://github.com/encode/httpx) and [Pydantic](https://pydantic.dev/) models for type-safe API interactions. +Stratix is an evaluation platform that allows you to benchmark AI models against various datasets and metrics. The Python SDK provides two HTTP clients (syncronous and asynchronous) powered by [httpx](https://github.com/encode/httpx) and [Pydantic](https://pydantic.dev/) models for type-safe API interactions. ## Quick Start @@ -16,27 +16,27 @@ Install the layerlens python sdk using the following command pip install layerlens --index-url https://sdk.layerlens.ai/package ``` -### Generate an api key on the atlas platform +### Generate an api key on the Stratix platform Login to your organization at [app.layerlens.ai](https://app.layerlens.ai) to generate an api key. Admin users of organizations can generate a keys in the settings page. Run this command to add your API key to your environment: ```bash -export LAYERLENS_ATLAS_API_KEY="YOUR_API_KEY" +export LAYERLENS_STRATIX_API_KEY="YOUR_API_KEY" ``` -### Running an evaluation on the atlas platform +### Running an evaluation on the Stratix platform Before triggering an evaluation using the sdk, login to your organization at [app.layerlens.ai](https://app.layerlens.ai) to ensure that the model and benchmark you are trying to evaluate has been added to your organizations dashboard. #### Using synchronous client ```python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client - client = Atlas() + client = Stratix() # --- Models replace with the model key you want to run model = client.models.get_by_key("openai/gpt-4o") @@ -61,11 +61,11 @@ from layerlens import Atlas ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def run_evaluation_async(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Model to use model = await client.models.get_by_key("openai/gpt-4o") diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index d64af0c..3002655 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -1,4 +1,4 @@ -# Table of Contents - Atlas Python SDK +# Table of Contents - Stratix Python SDK * [Introduction](README.md) @@ -12,10 +12,15 @@ * [Evaluations](api-reference/evaluations.md) * [Results](api-reference/results.md) * [Models & Benchmarks](api-reference/models-benchmarks.md) +* [Judges](api-reference/judges.md) +* [Traces](api-reference/traces.md) +* [Trace Evaluations](api-reference/trace-evaluations.md) +* [Judge Optimizations](api-reference/judge-optimizations.md) * [Error Handling](api-reference/errors.md) ## Code Examples * [Creating Evaluations](examples/creating-evaluations.md) +* [Judges and Traces](examples/judges-and-traces.md) * [Retrieving Results](examples/retrieving-results.md) * [Working with Timeouts](examples/timeouts.md) * [Advanced Usage Patterns](examples/advanced-usage.md) diff --git a/docs/api-reference/client.md b/docs/api-reference/client.md index 30a57c7..97f13bc 100644 --- a/docs/api-reference/client.md +++ b/docs/api-reference/client.md @@ -1,44 +1,44 @@ # Client Configuration -The `Atlas` (syncronous) and `AsyncAtlas` (asyncronous) classes are the main entry points for interacting with the LayerLens Atlas sdk. This page covers client initialization, configuration options, and usage patterns. +The `Stratix` (syncronous) and `AsyncStratix` (asyncronous) classes are the main entry points for interacting with the LayerLens Stratix sdk. This page covers client initialization, configuration options, and usage patterns. ## Basic Usage ### Syncronous Client ```python -from layerlens import Atlas +from layerlens import Stratix # Construct syncronous client -# Loads for api key from the "LAYERLENS_ATLAS_API_KEY" enviornment variable -client = Atlas() +# Loads for api key from the "LAYERLENS_STRATIX_API_KEY" enviornment variable +client = Stratix() # Explicit configuration -client = Atlas(api_key="your_api_key") +client = Stratix(api_key="your_api_key") ``` ### Asyncronous Client ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix # Construct async client -# Loads for api key from the "LAYERLENS_ATLAS_API_KEY" enviornment variable -client = AsyncAtlas() +# Loads for api key from the "LAYERLENS_STRATIX_API_KEY" enviornment variable +client = AsyncStratix() # Explicit configuration -client = AsyncAtlas(api_key="your_api_key") +client = AsyncStratix(api_key="your_api_key") ``` ## Constructor Parameters -### `Atlas(api_key, base_url, timeout)` and `AsyncAtlas(api_key, base_url, timeout)` +### `Stratix(api_key, base_url, timeout)` and `AsyncStratix(api_key, base_url, timeout)` | Parameter | Type | Required | Default | Description | | ---------- | -------------------------------- | -------- | ------------- | ----------------------------- | -| `api_key` | `str \| None` | Yes\* | `None` | Your LayerLens Atlas API key | -| `base_url` | `str \| httpx.URL \| None` | No | Atlas API URL | Custom API base URL | +| `api_key` | `str \| None` | Yes\* | `None` | Your LayerLens Stratix API key | +| `base_url` | `str \| httpx.URL \| None` | No | Stratix API URL | Custom API base URL | | `timeout` | `float \| httpx.Timeout \| None` | No | 10 minutes | Request timeout configuration | \*Required unless set via environment variables @@ -48,7 +48,7 @@ client = AsyncAtlas(api_key="your_api_key") The client automatically loads configuration from these environment variables: ```bash -LAYERLENS_ATLAS_API_KEY="your_api_key_here" +LAYERLENS_STRATIX_API_KEY="your_api_key_here" ``` ## Timeout Configuration @@ -56,16 +56,16 @@ LAYERLENS_ATLAS_API_KEY="your_api_key_here" ### Simple Timeout ```python -from layerlens import Atlas +from layerlens import Stratix # 30-second timeout for all requests -client = Atlas(timeout=30.0) +client = Stratix(timeout=30.0) ``` ### Per-Request Timeout Override ```python -client = Atlas() +client = Stratix() # --- Models replace with the model name you want to run models = client.models.get(type="public", name="gpt-4o") diff --git a/docs/api-reference/errors.md b/docs/api-reference/errors.md index c590d3b..14d83a3 100644 --- a/docs/api-reference/errors.md +++ b/docs/api-reference/errors.md @@ -1,13 +1,13 @@ # Error Handling -The Atlas Python SDK provides a comprehensive exception hierarchy to help you handle different error conditions gracefully. This guide covers all available exception types and best practices for error handling. +The Stratix Python SDK provides a comprehensive exception hierarchy to help you handle different error conditions gracefully. This guide covers all available exception types and best practices for error handling. ## Exception Hierarchy -All Atlas exceptions inherit from the base `AtlasError` class: +All Stratix exceptions inherit from the base `StratixError` class: ``` -AtlasError +StratixError ├── APIError │ ├── APIConnectionError │ │ └── APITimeoutError @@ -27,18 +27,18 @@ AtlasError ### Base Exceptions -#### `AtlasError` +#### `StratixError` -Base exception for all Atlas-related errors. +Base exception for all Stratix-related errors. ```python import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -except layerlens.AtlasError as e: - print(f"Atlas error occurred: {e}") +except layerlens.StratixError as e: + print(f"Stratix error occurred: {e}") ``` #### `APIError` @@ -55,7 +55,7 @@ Base exception for all API-related errors. Contains additional context about the import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.APIError as e: print(f"API error: {e.message}") @@ -80,7 +80,7 @@ Raised when the client cannot connect to the API server. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.APIConnectionError as e: print("Connection failed - check your network connection") @@ -95,7 +95,7 @@ Raised when a request times out. import layerlens try: - client = layerlens.Atlas(timeout=0.2) # Very short timeout + client = layerlens.Stratix(timeout=0.2) # Very short timeout evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.APITimeoutError: print("Request timed out - try increasing timeout or check network") @@ -119,7 +119,7 @@ Request was malformed or contained invalid parameters. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() # Invalid parameters evaluation = client.evaluations.create(model="", benchmark="") except layerlens.BadRequestError as e: @@ -135,11 +135,11 @@ API key is missing, invalid, or expired. import layerlens try: - client = layerlens.Atlas(api_key="invalid_key") + client = layerlens.Stratix(api_key="invalid_key") evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.AuthenticationError: print("Authentication failed - check your API key") - print("Make sure LAYERLENS_ATLAS_API_KEY is set correctly") + print("Make sure LAYERLENS_STRATIX_API_KEY is set correctly") ``` #### `PermissionDeniedError` (403) @@ -150,7 +150,7 @@ Valid API key but insufficient permissions for the requested operation. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="restricted-model", benchmark="mmlu") except layerlens.PermissionDeniedError: print("Permission denied - check your organization/project access") @@ -165,11 +165,11 @@ Requested resource (model, benchmark, evaluation) does not exist. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="nonexistent-model", benchmark="mmlu") except layerlens.NotFoundError: print("Model or benchmark not found") - print("Check available models and benchmarks in the Atlas dashboard") + print("Check available models and benchmarks in the Stratix Dashboard") ``` #### `ConflictError` (409) @@ -180,7 +180,7 @@ Request conflicts with current resource state. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() # Some operation that conflicts with current state evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.ConflictError: @@ -195,7 +195,7 @@ Request parameters are valid but cannot be processed. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="invalid-benchmark") except layerlens.UnprocessableEntityError as e: print(f"Cannot process request: {e}") @@ -211,7 +211,7 @@ import layerlens import time try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.RateLimitError as e: print("Rate limit exceeded") @@ -233,7 +233,7 @@ Server-side error occurred. import layerlens try: - client = layerlens.Atlas() + client = layerlens.Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.InternalServerError as e: print(f"Server error: {e.status_code}") @@ -248,10 +248,10 @@ except layerlens.InternalServerError as e: ```python import layerlens import time -from layerlens import Atlas +from layerlens import Stratix def robust_create_evaluation(model: str, benchmark: str, max_retries: int = 3): - client = Atlas() + client = Stratix() for attempt in range(max_retries): try: @@ -307,10 +307,10 @@ def robust_create_evaluation(model: str, benchmark: str, max_retries: int = 3): ```python import layerlens -from layerlens import Atlas +from layerlens import Stratix def get_evaluation_results_with_fallback(evaluation_id: str): - client = Atlas() + client = Stratix() try: results = client.results.get(evaluation_id=evaluation_id) @@ -345,14 +345,14 @@ else: ```python import logging import layerlens -from layerlens import Atlas +from layerlens import Stratix # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def monitored_api_call(): - client = Atlas() + client = Stratix() try: logger.info("Creating evaluation...") @@ -391,14 +391,14 @@ def monitored_api_call(): ```python import layerlens from contextlib import contextmanager -from layerlens import Atlas +from layerlens import Stratix @contextmanager -def atlas_client(): - """Context manager for Atlas client with error handling""" +def stratix_client(): + """Context manager for Stratix client with error handling""" client = None try: - client = Atlas() + client = Stratix() yield client except layerlens.AuthenticationError: print("Authentication failed") @@ -412,11 +412,11 @@ def atlas_client(): # Usage try: - with atlas_client() as client: + with stratix_client() as client: evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") results = client.results.get(evaluation_id=evaluation.id) -except layerlens.AtlasError: - print("Atlas operation failed") +except layerlens.StratixError: + print("Stratix operation failed") ``` ## Error Response Details @@ -425,10 +425,10 @@ except layerlens.AtlasError: ```python import layerlens -from layerlens import Atlas +from layerlens import Stratix try: - client = Atlas() + client = Stratix() evaluation = client.evaluations.create(model="invalid", benchmark="invalid") except layerlens.APIStatusError as e: print(f"Status Code: {e.status_code}") @@ -443,7 +443,7 @@ except layerlens.APIStatusError as e: ```python import layerlens -from layerlens import Atlas +from layerlens import Stratix def extract_error_info(error: layerlens.APIError): info = { @@ -466,7 +466,7 @@ def extract_error_info(error: layerlens.APIError): # Usage try: - client = Atlas() + client = Stratix() evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.APIError as e: error_info = extract_error_info(e) @@ -479,19 +479,19 @@ except layerlens.APIError as e: import pytest import layerlens from unittest.mock import Mock, patch -from layerlens import Atlas +from layerlens import Stratix def test_authentication_error_handling(): """Test that authentication errors are handled properly""" - with patch('layerlens.Atlas') as mock_atlas: - mock_atlas.side_effect = layerlens.AuthenticationError( + with patch('layerlens.Stratix') as mock_stratix: + mock_stratix.side_effect = layerlens.AuthenticationError( "Invalid API key", request=Mock(), response=Mock() ) with pytest.raises(layerlens.AuthenticationError): - client = Atlas() + client = Stratix() client.evaluations.create(model="gpt-4", benchmark="mmlu") def test_rate_limit_retry(): @@ -507,8 +507,8 @@ def test_rate_limit_retry(): ```python # Missing API key try: - client = Atlas(api_key=None) -except layerlens.AtlasError as e: + client = Stratix(api_key=None) +except layerlens.StratixError as e: print(f"Configuration error: {e}") ``` @@ -517,7 +517,7 @@ except layerlens.AtlasError as e: ```python # Connection timeout try: - client = Atlas(timeout=0.1) # Very short timeout + client = Stratix(timeout=0.1) # Very short timeout evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") except layerlens.APITimeoutError: print("Request timed out") @@ -538,7 +538,7 @@ except layerlens.APIConnectionError: import time import random import layerlens -from layerlens import Atlas +from layerlens import Stratix def exponential_backoff_retry(func, max_retries=3, base_delay=1): """Retry function with exponential backoff""" @@ -555,7 +555,7 @@ def exponential_backoff_retry(func, max_retries=3, base_delay=1): # Usage def create_evaluation(): - client = Atlas() + client = Stratix() return client.evaluations.create(model="gpt-4", benchmark="mmlu") evaluation = exponential_backoff_retry(create_evaluation) @@ -566,7 +566,7 @@ evaluation = exponential_backoff_retry(create_evaluation) ```python import time from enum import Enum -from layerlens import Atlas +from layerlens import Stratix import layerlens class CircuitState(Enum): @@ -609,7 +609,7 @@ class CircuitBreaker: # Usage breaker = CircuitBreaker() -client = Atlas() +client = Stratix() try: evaluation = breaker.call( diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index ce75bbc..9fa5184 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -1,6 +1,6 @@ # Evaluations -The `evaluations` resource on the atlas client allows you to create and manage evaluations against various benchmarks for your organization on the atlas platform. This is one of the core functionalities of the Atlas platform. +The `evaluations` resource on the Stratix client allows you to create and manage evaluations against various benchmarks for your organization on the Stratix platform. This is one of the core functionalities of the Stratix platform. ## Overview @@ -15,10 +15,10 @@ The below example trigger evaluations using `gpt-4o` against `simpleQA`. Below is an example showing how to trigger an evaluation, waiting for it to complete and finally fetching the evaluations results. ```python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client (API key from env or inline) -client = Atlas() +client = Stratix() # --- Models models = client.models.get(type="public", name="gpt-4o") @@ -62,12 +62,12 @@ if evaluation.is_success: ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get(type="public", name="gpt-4o") @@ -95,7 +95,7 @@ if __name__ == "__main__": ## Methods -Both the `Atlas` (synchronous) and `AsyncAtlas` (asynchronous) clients support the following methods. +Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. ### `create(model, benchmark, timeout=None)` @@ -147,9 +147,9 @@ Returns an `Evaluation` object if found, `None` if the evaluation does not exist #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Retrieve an evaluation by ID evaluation_id = "eval_abc123xyz" @@ -159,11 +159,11 @@ evaluation = client.evaluations.get_by_id(evaluation_id) #### Async Usage ```python -from layerlens import AsyncAtlas +from layerlens import AsyncStratix import asyncio async def get_evaluation(): - client = AsyncAtlas() + client = AsyncStratix() evaluation = await client.evaluations.get_by_id("eval_abc123xyz") if evaluation: @@ -217,9 +217,9 @@ Returns a `ResultsResponse` object containing results and pagination metadata, o #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get evaluation first evaluation = client.evaluations.get_by_id("eval_12345") @@ -245,9 +245,9 @@ Returns a list of `Result` objects containing all results for the evaluation. #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get an evaluation evaluation = client.evaluations.get_by_id("eval_12345") @@ -281,11 +281,11 @@ Returns a `ResultsResponse` object containing results and pagination metadata, o #### Example ```python -from layerlens import AsyncAtlas +from layerlens import AsyncStratix import asyncio async def fetch_evaluation_results(): - client = AsyncAtlas() + client = AsyncStratix() # Get an evaluation evaluation = await client.evaluations.get_by_id("eval_12345") @@ -319,9 +319,9 @@ Returns a list of `Result` objects containing all results for the evaluation. #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get an evaluation evaluation = client.evaluations.get_by_id("eval_12345") @@ -350,11 +350,11 @@ Returns a list of `Result` objects containing all results for the evaluation. #### Example ```python -from layerlens import AsyncAtlas +from layerlens import AsyncStratix import asyncio async def fetch_all_evaluation_results(): - client = AsyncAtlas() + client = AsyncStratix() # Get an evaluation evaluation = await client.evaluations.get_by_id("eval_12345") diff --git a/docs/api-reference/judge-optimizations.md b/docs/api-reference/judge-optimizations.md new file mode 100644 index 0000000..d03eb99 --- /dev/null +++ b/docs/api-reference/judge-optimizations.md @@ -0,0 +1,279 @@ +# Judge Optimizations + +The `judge_optimizations` resource on the Stratix client allows you to optimize a judge's evaluation criteria using automated prompt engineering. This can improve the accuracy and consistency of your judge's evaluations. + +## Overview + +A judge optimization run takes an existing judge and refines its evaluation goal through automated testing and prompt engineering. You can estimate costs before running, monitor progress, and apply successful optimizations to update the judge. + +### Using Synchronous Client + +```python +from layerlens import Stratix + +client = Stratix() + +# Estimate cost before running +estimate = client.judge_optimizations.estimate( + judge_id="judge-123", + budget="medium", +) +print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + +# Start an optimization run +run = client.judge_optimizations.create( + judge_id="judge-123", + budget="medium", +) + +# Poll for completion +import time +for _ in range(60): + optimization = client.judge_optimizations.get(run.id) + if optimization.status.value in ("success", "failure"): + break + time.sleep(5) + +# Apply successful results +if optimization.status.value == "success": + result = client.judge_optimizations.apply(run.id) + print(f"New version: v{result.new_version}") +``` + +### Using Async Client + +```python +import asyncio +from layerlens import AsyncStratix + +async def main(): + client = AsyncStratix() + + run = await client.judge_optimizations.create( + judge_id="judge-123", + budget="medium", + ) + + # Poll for completion + import time + for _ in range(60): + optimization = await client.judge_optimizations.get(run.id) + if optimization.status.value in ("success", "failure"): + break + await asyncio.sleep(5) + + if optimization.status.value == "success": + result = await client.judge_optimizations.apply(run.id) + print(f"New version: v{result.new_version}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Methods + +Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. + +### `estimate(judge_id, budget="medium", timeout=None)` + +Estimates the cost of running an optimization on a judge before actually executing it. + +#### Parameters + +| Parameter | Type | Required | Description | +| ---------- | -------------------------------- | -------- | ------------------------------------------ | +| `judge_id` | `str` | Yes | ID of the judge to optimize | +| `budget` | `str` | No | Optimization budget: "light", "medium", or "heavy" (default: "medium") | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns an `EstimateJudgeOptimizationCostResponse` object if successful, `None` otherwise. + +#### Example + +```python +estimate = client.judge_optimizations.estimate( + judge_id="judge-123", + budget="heavy", +) +if estimate: + print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + print(f"Annotations: {estimate.annotation_count}") + print(f"Budget: {estimate.budget}") +``` + +### `create(judge_id, budget="medium", timeout=None)` + +Starts a new optimization run for a judge. The optimization runs asynchronously on the server. + +#### Parameters + +| Parameter | Type | Required | Description | +| ---------- | -------------------------------- | -------- | ------------------------------------------ | +| `judge_id` | `str` | Yes | ID of the judge to optimize | +| `budget` | `str` | No | Optimization budget: "light", "medium", or "heavy" (default: "medium") | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `CreateJudgeOptimizationRunResponse` object if successful, `None` otherwise. + +#### Example + +```python +run = client.judge_optimizations.create( + judge_id="judge-123", + budget="medium", +) +print(f"Optimization {run.id}: {run.status}") +``` + +### `get(id, timeout=None)` + +Retrieves the current state of an optimization run by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ---------------------------------- | +| `id` | `str` | Yes | The unique optimization run ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `JudgeOptimizationRun` object if found, `None` otherwise. + +#### Example + +```python +optimization = client.judge_optimizations.get("opt-run-123") +if optimization: + print(f"Status: {optimization.status}") + if optimization.status.value == "success": + print(f"Baseline accuracy: {optimization.baseline_accuracy}") + print(f"Optimized accuracy: {optimization.optimized_accuracy}") +``` + +### `get_many(judge_id=None, page=None, page_size=None, timeout=None)` + +Retrieves multiple optimization runs with optional filtering and pagination. + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------- | -------------------------------- | -------- | ------------------------------------------------------ | +| `judge_id` | `str \| None` | No | Filter by judge | +| `page` | `int \| None` | No | Page number (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of runs per page (default: 20, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `JudgeOptimizationRunsResponse` object containing: + +- `optimization_runs`: List of `JudgeOptimizationRun` objects +- `count`: Number of runs in this page +- `total`: Total number of matching runs + +Returns `None` if the request fails. + +#### Example + +```python +# List all optimization runs +response = client.judge_optimizations.get_many() +print(f"Total runs: {response.total}") + +# Filter by judge +response = client.judge_optimizations.get_many(judge_id="judge-123") +for run in response.optimization_runs: + print(f" {run.id}: {run.status} (budget: {run.budget})") +``` + +### `apply(id, timeout=None)` + +Applies the results of a successful optimization run to the judge, updating its evaluation goal and creating a new judge version. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ---------------------------------- | +| `id` | `str` | Yes | The unique optimization run ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns an `ApplyJudgeOptimizationResultResponse` object if successful, `None` otherwise. + +#### Example + +```python +result = client.judge_optimizations.apply("opt-run-123") +if result: + print(f"Applied to judge {result.judge_id}") + print(f"New version: v{result.new_version}") + print(f"{result.message}") +``` + +## Response Objects + +### CreateJudgeOptimizationRunResponse Properties + +| Property | Type | Description | +| ---------- | ----- | ---------------------------------- | +| `id` | `str` | Unique optimization run identifier | +| `judge_id` | `str` | ID of the judge being optimized | +| `budget` | `str` | Optimization budget level | +| `status` | `str` | Initial status of the run | + +### JudgeOptimizationRun Properties + +| Property | Type | Description | +| -------------------- | ----------------------------- | --------------------------------------------- | +| `id` | `str` | Unique optimization run identifier | +| `judge_id` | `str` | ID of the judge being optimized | +| `status` | `OptimizationRunStatus` | Current status (pending, in_progress, success, failure) | +| `status_description` | `str \| None` | Human-readable status description | +| `budget` | `OptimizationBudget` | Budget level (light, medium, heavy) | +| `annotation_count` | `int` | Number of annotations used | +| `baseline_accuracy` | `float \| None` | Accuracy before optimization | +| `optimized_accuracy` | `float \| None` | Accuracy after optimization | +| `original_goal` | `str \| None` | Original evaluation goal | +| `optimized_goal` | `str \| None` | Optimized evaluation goal | +| `estimated_cost` | `float` | Estimated cost in dollars | +| `actual_cost` | `float` | Actual cost incurred | +| `created_at` | `str` | ISO 8601 creation timestamp | +| `started_at` | `str \| None` | When optimization started | +| `finished_at` | `str \| None` | When optimization finished | +| `applied_at` | `str \| None` | When results were applied to the judge | +| `applied_version` | `int \| None` | Judge version created by applying results | + +### EstimateJudgeOptimizationCostResponse Properties + +| Property | Type | Description | +| ------------------ | ------- | ---------------------------------- | +| `estimated_cost` | `float` | Estimated cost in dollars | +| `annotation_count` | `int` | Number of annotations to process | +| `budget` | `str` | Budget level used for estimate | + +### ApplyJudgeOptimizationResultResponse Properties + +| Property | Type | Description | +| ------------- | ----- | ---------------------------------------------- | +| `judge_id` | `str` | ID of the updated judge | +| `new_version` | `int` | New version number of the judge | +| `message` | `str` | Confirmation message | + +## Optimization Budgets + +| Budget | Description | +| ---------- | ----------------------------------------------------------------- | +| `"light"` | Faster, lower cost. Good for quick iterations. | +| `"medium"` | Balanced cost and thoroughness. Recommended default. | +| `"heavy"` | Most thorough optimization. Higher cost but potentially better results. | + +## Next Steps + +- Learn about [Judges](judges.md) to create judges for optimization +- Learn about [Trace Evaluations](trace-evaluations.md) to evaluate traces with optimized judges diff --git a/docs/api-reference/judges.md b/docs/api-reference/judges.md new file mode 100644 index 0000000..e7b4906 --- /dev/null +++ b/docs/api-reference/judges.md @@ -0,0 +1,213 @@ +# Judges + +The `judges` resource on the Stratix client allows you to create and manage LLM-based judges for evaluating traces. Judges define evaluation criteria that can be run against traces to assess quality, correctness, and other properties. + +## Overview + +A judge encapsulates an evaluation goal and is backed by a specific LLM model. Once created, judges can be run against traces via the trace evaluations resource to produce scored results. + +### Using Synchronous Client + +```python +from layerlens import Stratix + +client = Stratix() + +# Fetch a model to use for the judge +models = client.models.get(type="public", name="gpt-4o") +model = models[0] + +# Create a judge +judge = client.judges.create( + name="Code Quality Judge", + evaluation_goal="Evaluate the quality of code output including correctness and style", + model_id=model.id, +) + +print(f"Created judge: {judge.name} (v{judge.version})") + +# List all judges +response = client.judges.get_many() +for j in response.judges: + print(f" {j.name}: {j.run_count} runs") +``` + +### Using Async Client + +```python +import asyncio +from layerlens import AsyncStratix + +async def main(): + client = AsyncStratix() + + judge = await client.judges.create( + name="Code Quality Judge", + evaluation_goal="Evaluate the quality of code output including correctness and style", + model_id="model-abc123", + ) + + print(f"Created judge: {judge.name} (v{judge.version})") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Methods + +Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. + +### `create(name, evaluation_goal, model_id=None, timeout=None)` + +Creates a new judge with the specified evaluation criteria. + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------------- | -------------------------------- | -------- | -------------------------------------------- | +| `name` | `str` | Yes | Display name for the judge | +| `evaluation_goal` | `str` | Yes | Description of what the judge should evaluate | +| `model_id` | `str \| None` | Yes* | ID of the LLM model to use (required by API)| +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `Judge` object if successful, `None` if the judge could not be created. + +#### Example + +```python +judge = client.judges.create( + name="Accuracy Judge", + evaluation_goal="Evaluate whether the response is factually accurate", + model_id="model-abc123", +) +``` + +### `get(id, timeout=None)` + +Retrieves a judge by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ------------------------ | +| `id` | `str` | Yes | The unique judge ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `Judge` object if found, `None` otherwise. + +### `get_many(page=None, page_size=None, timeout=None)` + +Retrieves multiple judges with pagination. + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------- | -------------------------------- | -------- | -------------------------------------------------- | +| `page` | `int \| None` | No | Page number (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of judges per page (default: 100, max: 500) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `JudgesResponse` object containing: + +- `judges`: List of `Judge` objects +- `count`: Number of judges in this page +- `total_count`: Total number of judges + +Returns `None` if the request fails. + +#### Example + +```python +# Get first page +response = client.judges.get_many() +print(f"Total judges: {response.total_count}") + +# Get specific page +response = client.judges.get_many(page=2, page_size=50) +``` + +### `update(id, name=None, evaluation_goal=None, model_id=None, timeout=None)` + +Updates an existing judge. Only provided fields are modified; omitted fields remain unchanged. Updating a judge creates a new version. + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------------- | -------------------------------- | -------- | -------------------------------------------- | +| `id` | `str` | Yes | The unique judge ID | +| `name` | `str \| None` | No | Updated display name | +| `evaluation_goal` | `str \| None` | No | Updated evaluation criteria | +| `model_id` | `str \| None` | No | Updated model ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns an `UpdateJudgeResponse` if successful, `None` otherwise. + +#### Example + +```python +updated = client.judges.update( + "judge-123", + evaluation_goal="Evaluate code for correctness, readability, and security", +) +``` + +### `delete(id, timeout=None)` + +Deletes a judge by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ------------------------ | +| `id` | `str` | Yes | The unique judge ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `DeleteJudgeResponse` if successful, `None` otherwise. + +## Response Objects + +### Judge Object Properties + +| Property | Type | Description | +| ----------------- | --------------------- | ---------------------------------------- | +| `id` | `str` | Unique judge identifier | +| `organization_id` | `str` | Organization the judge belongs to | +| `project_id` | `str` | Project the judge belongs to | +| `name` | `str` | Display name | +| `evaluation_goal` | `str` | Description of evaluation criteria | +| `model_id` | `str \| None` | ID of the backing LLM model | +| `model_name` | `str \| None` | Name of the backing LLM model | +| `model_company` | `str \| None` | Company that provides the model | +| `version` | `int` | Current version number | +| `run_count` | `int` | Number of times this judge has been run | +| `created_at` | `str` | ISO 8601 creation timestamp | +| `updated_at` | `str \| None` | ISO 8601 last update timestamp | +| `versions` | `List[JudgeVersion]` | Version history | + +### JudgeVersion Object Properties + +| Property | Type | Description | +| ----------------- | ------------- | ---------------------------------- | +| `version` | `int` | Version number | +| `name` | `str` | Name at this version | +| `evaluation_goal` | `str` | Evaluation goal at this version | +| `model_id` | `str \| None` | Model ID at this version | +| `model_name` | `str \| None` | Model name at this version | +| `model_company` | `str \| None` | Model company at this version | +| `updated_at` | `str \| None` | When this version was created | +| `updated_by` | `str \| None` | Who created this version | + +## Next Steps + +- Learn about [Traces](traces.md) to upload data for evaluation +- Learn about [Trace Evaluations](trace-evaluations.md) to run judges against traces diff --git a/docs/api-reference/models-benchmarks.md b/docs/api-reference/models-benchmarks.md index 6844f12..1681b75 100644 --- a/docs/api-reference/models-benchmarks.md +++ b/docs/api-reference/models-benchmarks.md @@ -4,27 +4,27 @@ This page provides reference information about accessing the available models an ## Overview -Running evaluations on the atlas platform require a model and benchmark to be selected. The availability of models and benchmarks depends on your organizations configuration. +Running evaluations on the Stratix platform require a model and benchmark to be selected. The availability of models and benchmarks depends on your organizations configuration. > Before running the below examples ensure the model and benchmark being run are present on your organiztion. ### Finding Available Models and Benchmarks -#### 1. Using the Atlas Dashboard +#### 1. Using the Stratix Dashboard The most reliable way to find available models and benchmarks: -1. Log into your Atlas dashboard. +1. Log into your Stratix Dashboard. 2. Navigate to the evaluation creation page. 3. View dropdown lists of available models and benchmarks. #### 2. Using the python sdk ```python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client (API key from env or inline) -client = Atlas() +client = Stratix() # --- Models models = client.models.get() @@ -37,7 +37,7 @@ benchmarks = client.benchmarks.get() ### `get(type=None, name=None, companies=None, regions=None, licenses=None, timeout=None)` -Retrieves a list of available models with optional filtering parameters. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a list of available models with optional filtering parameters. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters @@ -67,7 +67,7 @@ Each `Model` object in the returned list contains: ### `get_by_id(id, timeout=None)` -Retrieves a specific model by its unique identifier. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a specific model by its unique identifier. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters @@ -82,7 +82,7 @@ Returns an `Optional[Model]` - a single `Model` object if found, or `None` if th ### `get_by_key(key, timeout=None)` -Retrieves a specific model by its unique key. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a specific model by its unique key. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters @@ -99,7 +99,7 @@ Returns an `Optional[Model]` - a single `Model` object if found, or `None` if th ### `get(type=None, name=None, timeout=None)` -Retrieves a list of available benchmarks with optional filtering parameters. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a list of available benchmarks with optional filtering parameters. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters @@ -127,7 +127,7 @@ Each `Benchmark` object in the returned list contains: ### `get_by_id(id, timeout=None)` -Retrieves a specific benchmark by its unique identifier. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a specific benchmark by its unique identifier. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters @@ -142,7 +142,7 @@ Returns an `Optional[Benchmark]` - a single `Benchmark` object if found, or `Non ### `get_by_key(key, timeout=None)` -Retrieves a specific benchmark by its unique key. Both the `Atlas` and `AsyncAtlas` clients have this method. +Retrieves a specific benchmark by its unique key. Both the `Stratix` and `AsyncStratix` clients have this method. #### Parameters diff --git a/docs/api-reference/results.md b/docs/api-reference/results.md index 33ed755..5540c88 100644 --- a/docs/api-reference/results.md +++ b/docs/api-reference/results.md @@ -6,7 +6,7 @@ The `results` resource allows you to retrieve detailed results from completed or Results contain detailed information about each test case in an evaluation. -> Both the `Atlas` (synchronous) and `AsyncAtlas` (asynchronous) clients support the following methods. +> Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. ## Methods @@ -28,9 +28,9 @@ Returns a `List[Result]` containing all result objects across all pages. Returns #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get evaluation first evaluation = client.evaluations.get_by_id("eval_12345") @@ -46,11 +46,11 @@ else: #### Async Usage ```python -from layerlens import AsyncAtlas +from layerlens import AsyncStratix import asyncio async def get_all_results(): - client = AsyncAtlas() + client = AsyncStratix() # Get evaluation first evaluation = await client.evaluations.get_by_id("eval_12345") @@ -86,9 +86,9 @@ Returns a `List[Result]` containing all result objects across all pages. Returns #### Example ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get all results directly by evaluation ID all_results = client.results.get_all_by_id(evaluation_id="eval_12345") @@ -97,11 +97,11 @@ all_results = client.results.get_all_by_id(evaluation_id="eval_12345") #### Async Usage ```python -from layerlens import AsyncAtlas +from layerlens import AsyncStratix import asyncio async def get_all_results(): - client = AsyncAtlas() + client = AsyncStratix() # Get all results asynchronously all_results = await client.results.get_all_by_id(evaluation_id="eval_12345") @@ -144,9 +144,9 @@ The `ResultsResponse` object includes: ##### Basic Usage (All Results) ```python -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() # Get all results for a specific evaluation results_data = client.results.get(evaluation_id="eval_12345") @@ -289,10 +289,10 @@ Results can contain thousands of individual test cases. Consider using the async ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def fetch_results_async(): - async_client = AsyncAtlas() + async_client = AsyncStratix() # Get evaluation first evaluation = await async_client.evaluations.get_by_id("eval_12345") diff --git a/docs/api-reference/trace-evaluations.md b/docs/api-reference/trace-evaluations.md new file mode 100644 index 0000000..0e88b61 --- /dev/null +++ b/docs/api-reference/trace-evaluations.md @@ -0,0 +1,266 @@ +# Trace Evaluations + +The `trace_evaluations` resource on the Stratix client allows you to run judges against traces and retrieve scored results. This is how you assess the quality of your trace data using the evaluation criteria defined in your judges. + +## Overview + +A trace evaluation runs a specific judge against a specific trace, producing a scored result with reasoning and step-by-step analysis. You can estimate costs before running evaluations and retrieve detailed results afterwards. + +### Using Synchronous Client + +```python +from layerlens import Stratix + +client = Stratix() + +# Estimate cost before running +estimate = client.trace_evaluations.estimate_cost( + trace_ids=["trace-1", "trace-2"], + judge_id="judge-123", +) +print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + +# Run a judge on a trace +evaluation = client.trace_evaluations.create( + trace_id="trace-1", + judge_id="judge-123", +) + +# Get results +results = client.trace_evaluations.get_results(evaluation.id) +for result in results.results: + print(f"Score: {result.score}, Passed: {result.passed}") + print(f"Reasoning: {result.reasoning}") +``` + +### Using Async Client + +```python +import asyncio +from layerlens import AsyncStratix + +async def main(): + client = AsyncStratix() + + evaluation = await client.trace_evaluations.create( + trace_id="trace-1", + judge_id="judge-123", + ) + + results = await client.trace_evaluations.get_results(evaluation.id) + for result in results.results: + print(f"Score: {result.score}, Passed: {result.passed}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Methods + +Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. + +### `create(trace_id, judge_id, timeout=None)` + +Runs a judge against a trace, creating a new trace evaluation. + +#### Parameters + +| Parameter | Type | Required | Description | +| ---------- | -------------------------------- | -------- | ------------------------------ | +| `trace_id` | `str` | Yes | ID of the trace to evaluate | +| `judge_id` | `str` | Yes | ID of the judge to run | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `TraceEvaluation` object if successful, `None` otherwise. + +#### Example + +```python +evaluation = client.trace_evaluations.create( + trace_id="trace-abc", + judge_id="judge-xyz", +) +print(f"Evaluation {evaluation.id}: {evaluation.status}") +``` + +### `get(id, timeout=None)` + +Retrieves a trace evaluation by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ---------------------------------- | +| `id` | `str` | Yes | The unique trace evaluation ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `TraceEvaluation` object if found, `None` otherwise. + +### `get_many(page=None, page_size=None, judge_id=None, trace_id=None, outcome=None, time_range=None, search=None, sort_by=None, sort_order=None, timeout=None)` + +Retrieves multiple trace evaluations with filtering and pagination. + +#### Parameters + +| Parameter | Type | Required | Description | +| ------------ | -------------------------------- | -------- | ---------------------------------------------------------- | +| `page` | `int \| None` | No | Page number (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of evaluations per page (default: 20, max: 100) | +| `judge_id` | `str \| None` | No | Filter by judge | +| `trace_id` | `str \| None` | No | Filter by trace | +| `outcome` | `str \| None` | No | Filter by outcome (e.g., "pass", "fail") | +| `time_range` | `str \| None` | No | Filter by time range (e.g., "7d", "30d") | +| `search` | `str \| None` | No | Search term to filter evaluations | +| `sort_by` | `str \| None` | No | Field to sort by (e.g., "created_at") | +| `sort_order` | `str \| None` | No | Sort direction: "asc" or "desc" | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `TraceEvaluationsResponse` object containing: + +- `trace_evaluations`: List of `TraceEvaluation` objects +- `count`: Number of evaluations in this page +- `total`: Total number of matching evaluations + +Returns `None` if the request fails. + +#### Example + +```python +# Get all evaluations +response = client.trace_evaluations.get_many() +print(f"Total: {response.total}") + +# Filtered by judge and outcome +response = client.trace_evaluations.get_many( + judge_id="judge-123", + outcome="pass", + sort_by="created_at", + sort_order="desc", +) +``` + +### `get_results(id, timeout=None)` + +Retrieves the detailed results of a completed trace evaluation, including scores, reasoning, and step-by-step analysis. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ---------------------------------- | +| `id` | `str` | Yes | The unique trace evaluation ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `TraceEvaluationResultsResponse` object containing: + +- `results`: List of `TraceEvaluationResult` objects + +Returns `None` if the request fails. + +#### Example + +```python +results_response = client.trace_evaluations.get_results("eval-123") +if results_response: + for result in results_response.results: + print(f"Score: {result.score}") + print(f"Passed: {result.passed}") + print(f"Reasoning: {result.reasoning}") + for step in result.steps: + print(f" Step {step.step}: {step.reasoning}") +``` + +### `estimate_cost(trace_ids, judge_id, timeout=None)` + +Estimates the cost of running a judge against a set of traces before actually executing the evaluations. + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------- | -------------------------------- | -------- | ----------------------------------- | +| `trace_ids` | `List[str]` | Yes | List of trace IDs to evaluate | +| `judge_id` | `str` | Yes | ID of the judge to run | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `CostEstimateResponse` object if successful, `None` otherwise. + +#### Example + +```python +estimate = client.trace_evaluations.estimate_cost( + trace_ids=["trace-1", "trace-2", "trace-3"], + judge_id="judge-123", +) +if estimate: + print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + print(f"Traces: {estimate.trace_count}") + print(f"Model: {estimate.model}") +``` + +## Response Objects + +### TraceEvaluation Object Properties + +| Property | Type | Description | +| ---------------- | ---------------------------- | ---------------------------------------- | +| `id` | `str` | Unique evaluation identifier | +| `trace_id` | `str` | ID of the evaluated trace | +| `judge_id` | `str` | ID of the judge used | +| `status` | `TraceEvaluationStatus` | Current status of the evaluation | +| `judge_snapshot` | `JudgeSnapshot \| None` | Snapshot of judge config at run time | +| `created_at` | `str \| None` | ISO 8601 creation timestamp | +| `started_at` | `str \| None` | When evaluation started | +| `finished_at` | `str \| None` | When evaluation finished | + +#### TraceEvaluationStatus + +| Status | Description | +| --------------- | ------------------------------------ | +| `"pending"` | Evaluation queued but not started | +| `"in_progress"` | Evaluation currently running | +| `"success"` | Evaluation completed successfully | +| `"failure"` | Evaluation failed | + +### TraceEvaluationResult Object Properties + +| Property | Type | Description | +| --------------------- | ----------------------------- | ----------------------------------------- | +| `id` | `str` | Unique result identifier | +| `trace_evaluation_id` | `str` | Parent evaluation ID | +| `trace_id` | `str` | ID of the evaluated trace | +| `judge_id` | `str` | ID of the judge used | +| `score` | `float \| None` | Numerical score | +| `passed` | `bool \| None` | Whether the trace passed the evaluation | +| `reasoning` | `str \| None` | Overall reasoning for the score | +| `steps` | `List[TraceEvaluationStep]` | Step-by-step reasoning | +| `model` | `str \| None` | Model used for evaluation | +| `turns` | `int \| None` | Number of turns in evaluation | +| `latency_ms` | `int \| None` | Evaluation latency in milliseconds | +| `prompt_tokens` | `int \| None` | Number of prompt tokens used | +| `completion_tokens` | `int \| None` | Number of completion tokens used | +| `total_cost` | `float \| None` | Total cost of the evaluation | +| `created_at` | `str \| None` | ISO 8601 creation timestamp | + +### CostEstimateResponse Properties + +| Property | Type | Description | +| ---------------- | ------- | ---------------------------------- | +| `estimated_cost` | `float` | Estimated cost in dollars | +| `input_tokens` | `int` | Estimated input tokens | +| `output_tokens` | `int` | Estimated output tokens | +| `model` | `str` | Model that would be used | +| `trace_count` | `int` | Number of traces to evaluate | + +## Next Steps + +- Learn about [Judges](judges.md) to create evaluation criteria +- Learn about [Traces](traces.md) to upload data for evaluation diff --git a/docs/api-reference/traces.md b/docs/api-reference/traces.md new file mode 100644 index 0000000..ce4bfd8 --- /dev/null +++ b/docs/api-reference/traces.md @@ -0,0 +1,229 @@ +# Traces + +The `traces` resource on the Stratix client allows you to upload, retrieve, and manage trace data. Traces are flexible JSON documents representing execution data that can be evaluated by judges. + +## Overview + +Traces capture execution data (e.g., LLM interactions, agent workflows) in JSON or JSONL format. Once uploaded, traces can be evaluated using judges to assess quality and correctness. + +### Using Synchronous Client + +```python +from layerlens import Stratix + +client = Stratix() + +# Upload a trace file +result = client.traces.upload("./traces.jsonl") +print(f"Uploaded {len(result.trace_ids)} traces") + +# List traces +response = client.traces.get_many() +for trace in response.traces: + print(f" {trace.id}: {trace.filename}") +``` + +### Using Async Client + +```python +import asyncio +from layerlens import AsyncStratix + +async def main(): + client = AsyncStratix() + + result = await client.traces.upload("./traces.jsonl") + print(f"Uploaded {len(result.trace_ids)} traces") + + response = await client.traces.get_many() + for trace in response.traces: + print(f" {trace.id}: {trace.filename}") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Methods + +Both the `Stratix` (synchronous) and `AsyncStratix` (asynchronous) clients support the following methods. + +### `upload(file_path, timeout=None)` + +Uploads a JSON or JSONL trace file. This is a three-step process handled automatically: + +1. Requests a presigned upload URL from the API +2. Uploads the file to S3 via the presigned URL +3. Creates trace records from the uploaded file + +#### Parameters + +| Parameter | Type | Required | Description | +| ----------- | -------------------------------- | -------- | --------------------------------------- | +| `file_path` | `str` | Yes | Path to the JSON or JSONL file | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `CreateTracesResponse` object if successful, `None` if the upload fails. + +#### Constraints + +- Maximum file size: 50 MB +- Supported formats: `.json`, `.jsonl` + +#### Example + +```python +# Upload a JSONL file +result = client.traces.upload("./my-traces.jsonl") +if result: + print(f"Uploaded {len(result.trace_ids)} traces") + +# Upload a JSON file +result = client.traces.upload("./single-trace.json") +``` + +### `get(id, timeout=None)` + +Retrieves a single trace by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ------------------------ | +| `id` | `str` | Yes | The unique trace ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `Trace` object if found, `None` otherwise. + +#### Example + +```python +trace = client.traces.get("trace-abc123") +if trace: + print(f"Trace: {trace.filename}") + print(f"Data keys: {list(trace.data.keys())}") +``` + +### `get_many(page=None, page_size=None, source=None, judge_id=None, status=None, time_range=None, search=None, sort_by=None, sort_order=None, timeout=None)` + +Retrieves multiple traces with filtering, searching, and pagination. + +#### Parameters + +| Parameter | Type | Required | Description | +| ------------ | -------------------------------- | -------- | --------------------------------------------------- | +| `page` | `int \| None` | No | Page number (1-based, defaults to 1) | +| `page_size` | `int \| None` | No | Number of traces per page (default: 100, max: 500) | +| `source` | `str \| None` | No | Filter by source (e.g., "upload") | +| `judge_id` | `str \| None` | No | Filter by associated judge | +| `status` | `str \| None` | No | Filter by evaluation status | +| `time_range` | `str \| None` | No | Filter by time range (e.g., "7d", "30d") | +| `search` | `str \| None` | No | Search term to filter traces | +| `sort_by` | `str \| None` | No | Field to sort by (e.g., "created_at") | +| `sort_order` | `str \| None` | No | Sort direction: "asc" or "desc" | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `TracesResponse` object containing: + +- `traces`: List of `TraceWithEvaluations` objects +- `count`: Number of traces in this page +- `total_count`: Total number of matching traces + +Returns `None` if the request fails. + +#### Example + +```python +# Get all traces +response = client.traces.get_many() +print(f"Total: {response.total_count}") + +# Filtered and sorted +response = client.traces.get_many( + source="upload", + time_range="7d", + sort_by="created_at", + sort_order="desc", + page_size=20, +) +``` + +### `delete(id, timeout=None)` + +Deletes a trace by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ------------------------ | +| `id` | `str` | Yes | The unique trace ID | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns `True` if the trace was deleted, `False` otherwise. + +### `get_sources(timeout=None)` + +Retrieves the list of available trace sources for the current project. + +#### Parameters + +| Parameter | Type | Required | Description | +| --------- | -------------------------------- | -------- | ------------------------ | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `List[str]` of source names. Returns an empty list if no sources are found. + +#### Example + +```python +sources = client.traces.get_sources() +print(f"Available sources: {sources}") +``` + +## Response Objects + +### Trace Object Properties + +| Property | Type | Description | +| ----------------- | ----------------- | ------------------------------------ | +| `id` | `str` | Unique trace identifier | +| `organization_id` | `str` | Organization the trace belongs to | +| `project_id` | `str` | Project the trace belongs to | +| `created_at` | `str` | ISO 8601 creation timestamp | +| `filename` | `str \| None` | Original filename of the upload | +| `data` | `Dict[str, Any]` | The trace data | +| `input` | `str \| None` | Input text or prompt | +| `integration_id` | `str \| None` | Integration that created this trace | + +### TraceWithEvaluations Object Properties + +Extends `Trace` with: + +| Property | Type | Description | +| ------------------- | ------------------------------ | ---------------------------------------- | +| `evaluations_count` | `int \| None` | Number of evaluations run on this trace | +| `last_evaluations` | `List[TraceEvaluationSummary]` | Most recent evaluation summaries | + +### TraceEvaluationSummary Properties + +| Property | Type | Description | +| --------------- | ------------- | ---------------------------------- | +| `judge_id` | `str` | ID of the judge used | +| `judge_name` | `str \| None` | Name of the judge | +| `judge_version` | `int \| None` | Version of the judge | +| `created_at` | `str \| None` | When the evaluation was run | +| `passed` | `bool \| None`| Whether the trace passed | + +## Next Steps + +- Learn about [Judges](judges.md) to create evaluation criteria +- Learn about [Trace Evaluations](trace-evaluations.md) to run judges against traces diff --git a/docs/examples/creating-evaluations.md b/docs/examples/creating-evaluations.md index 8b4c9e6..ffec8cf 100644 --- a/docs/examples/creating-evaluations.md +++ b/docs/examples/creating-evaluations.md @@ -1,6 +1,6 @@ # Creating Evaluations -Examples for creating evaluations on the Atlas platform using the Layerlens python sdk. +Examples for creating evaluations on the Stratix platform using the Layerlens python sdk. > Before running the below examples ensure the model and benchmark being run are present on your organiztion. @@ -11,10 +11,10 @@ Examples for creating evaluations on the Atlas platform using the Layerlens pyth Below is an example showing how to trigger an evaluation, waiting for it to complete and finally fetching the evaluations results. ```python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client (API key from env or inline) -client = Atlas() +client = Stratix() # --- Models replace with the model key you want to run model = client.models.get_by_key("openai/gpt-4o") @@ -54,12 +54,12 @@ if evaluation.is_success: ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Model to use model = await client.models.get_by_key("openai/gpt-4o") @@ -94,10 +94,10 @@ if __name__ == "__main__": ## Error Handling ```python -from layerlens import Atlas +from layerlens import Stratix import layerlens -client = Atlas() +client = Stratix() try: models = client.models.get() @@ -121,7 +121,7 @@ except layerlens.APIError as e: ```python import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def create_and_run_evaluation(client, model, benchmark, eval_number): @@ -153,7 +153,7 @@ async def create_and_run_evaluation(client, model, benchmark, eval_number): async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get() @@ -188,7 +188,7 @@ if __name__ == "__main__": import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def fetch_evaluation_results(client, evaluation_id): @@ -208,7 +208,7 @@ async def fetch_evaluation_results(client, evaluation_id): async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # List of example evaluation IDs to fetch diff --git a/docs/examples/judges-and-traces.md b/docs/examples/judges-and-traces.md new file mode 100644 index 0000000..8ac976e --- /dev/null +++ b/docs/examples/judges-and-traces.md @@ -0,0 +1,275 @@ +# Judges and Traces + +Examples for working with judges, traces, and trace evaluations on the Stratix platform using the Layerlens Python SDK. + +## Creating and Managing Judges + +### Basic Judge CRUD + +```python +from layerlens import Stratix + +client = Stratix() + +# Fetch a model to use for the judge +models = client.models.get(type="public", name="gpt-4o") +model = models[0] + +# Create a judge (model_id is required) +judge = client.judges.create( + name="Code Quality Judge", + evaluation_goal="Evaluate the quality of code output including correctness, readability, and style", + model_id=model.id, +) +print(f"Created judge: {judge.name} (v{judge.version})") + +# Get a judge by ID +judge = client.judges.get(judge.id) + +# List all judges with pagination +response = client.judges.get_many(page=1, page_size=50) +for j in response.judges: + print(f" {j.name}: v{j.version}, {j.run_count} runs") + +# Update a judge (creates a new version) +client.judges.update( + judge.id, + evaluation_goal="Evaluate code for correctness, readability, style, and security", +) + +# Delete a judge +client.judges.delete(judge.id) +``` + +## Uploading and Managing Traces + +### Upload Trace Files + +```python +from layerlens import Stratix + +client = Stratix() + +# Upload a JSONL file containing multiple traces +result = client.traces.upload("./traces.jsonl") +print(f"Uploaded {len(result.trace_ids)} traces") + +# Upload a single JSON trace +result = client.traces.upload("./single-trace.json") +``` + +### Browse and Filter Traces + +```python +from layerlens import Stratix + +client = Stratix() + +# List all traces +response = client.traces.get_many() +print(f"Total traces: {response.total_count}") + +# Filter by time range and sort +response = client.traces.get_many( + time_range="7d", + sort_by="created_at", + sort_order="desc", +) + +# Search traces +response = client.traces.get_many(search="authentication") + +# Get available sources +sources = client.traces.get_sources() +print(f"Sources: {sources}") +``` + +### Get Trace Details + +```python +from layerlens import Stratix + +client = Stratix() + +trace = client.traces.get("trace-abc123") +if trace: + print(f"Filename: {trace.filename}") + print(f"Created: {trace.created_at}") + print(f"Data keys: {list(trace.data.keys())}") +``` + +## Running Trace Evaluations + +### Estimate Cost Before Running + +```python +from layerlens import Stratix + +client = Stratix() + +# Get trace IDs to evaluate +traces_response = client.traces.get_many(page_size=10) +trace_ids = [t.id for t in traces_response.traces] + +# Estimate cost +estimate = client.trace_evaluations.estimate_cost( + trace_ids=trace_ids, + judge_id="judge-123", +) +print(f"Cost for {estimate.trace_count} traces: ${estimate.estimated_cost:.4f}") +print(f"Model: {estimate.model}") +``` + +### Run a Judge on a Trace + +```python +import time +from layerlens import Stratix + +client = Stratix() + +# Create an evaluation +evaluation = client.trace_evaluations.create( + trace_id="trace-abc", + judge_id="judge-xyz", +) +print(f"Evaluation {evaluation.id}: {evaluation.status}") + +# Wait for evaluation to complete (evaluations run asynchronously on the server) +for _ in range(30): + evaluation = client.trace_evaluations.get(evaluation.id) + if evaluation.status.value in ("success", "failure"): + break + time.sleep(2) + +# Get results (only available after evaluation completes) +try: + results_response = client.trace_evaluations.get_results(evaluation.id) + if results_response: + for result in results_response.results: + print(f"Score: {result.score}, Passed: {result.passed}") + print(f"Reasoning: {result.reasoning}") + print(f"Latency: {result.latency_ms}ms, Cost: ${result.total_cost:.4f}") + for step in result.steps: + print(f" Step {step.step}: {step.reasoning}") +except Exception: + print("Results not available yet") +``` + +### Browse Evaluation Results + +```python +from layerlens import Stratix + +client = Stratix() + +# List all evaluations +response = client.trace_evaluations.get_many() +print(f"Total evaluations: {response.total}") + +# Filter by judge and outcome +response = client.trace_evaluations.get_many( + judge_id="judge-123", + outcome="pass", + sort_by="created_at", + sort_order="desc", +) + +# Filter by trace +response = client.trace_evaluations.get_many( + trace_id="trace-abc", +) +``` + +## Async Workflows + +### Run Evaluations Concurrently + +```python +import asyncio +from layerlens import Stratix, AsyncStratix + +async def main(): + # Fetch a model for judge creation + sync_client = Stratix() + models = sync_client.models.get(type="public", name="gpt-4o") + model = models[0] + + client = AsyncStratix() + + # Create a judge (model_id is required) + judge = await client.judges.create( + name="Response Quality Judge", + evaluation_goal="Evaluate whether the response is accurate and well-structured", + model_id=model.id, + ) + + # Upload traces + result = await client.traces.upload("./traces.jsonl") + print(f"Uploaded {len(result.trace_ids)} traces") + + # Get traces to evaluate + traces_response = await client.traces.get_many(page_size=5) + trace_ids = [t.id for t in traces_response.traces] + + # Run evaluations concurrently + tasks = [ + client.trace_evaluations.create(trace_id=tid, judge_id=judge.id) + for tid in trace_ids + ] + evaluations = await asyncio.gather(*tasks) + + for evaluation in evaluations: + if evaluation: + print(f"Evaluation {evaluation.id}: {evaluation.status}") + + # Wait for evaluations to complete, then fetch results + await asyncio.sleep(10) + for evaluation in evaluations: + if not evaluation: + continue + try: + results_response = await client.trace_evaluations.get_results(evaluation.id) + if results_response: + for result in results_response.results: + print(f"Score: {result.score}, Passed: {result.passed}") + except Exception: + print(f"Evaluation {evaluation.id}: results not available yet") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Error Handling + +```python +from layerlens import Stratix +import layerlens + +client = Stratix() + +try: + # Fetch a model for the judge + models = client.models.get(type="public", name="gpt-4o") + model = models[0] + + judge = client.judges.create( + name="My Judge", + evaluation_goal="Evaluate output quality", + model_id=model.id, + ) + + evaluation = client.trace_evaluations.create( + trace_id="trace-abc", + judge_id=judge.id, + ) + +except layerlens.AuthenticationError: + print("Check your API key") +except layerlens.NotFoundError: + print("Trace or judge not found") +except layerlens.BadRequestError as e: + print(f"Invalid request: {e}") +except layerlens.APIError as e: + print(f"API error: {e}") +``` diff --git a/docs/security/api-key-management.md b/docs/security/api-key-management.md index bf5242f..bb059ba 100644 --- a/docs/security/api-key-management.md +++ b/docs/security/api-key-management.md @@ -1,12 +1,12 @@ # API Key Management -This guide covers best practices for securely managing your Atlas API keys throughout the development lifecycle. +This guide covers best practices for securely managing your Stratix API keys throughout the development lifecycle. ## API Key Security Fundamentals ### What Makes API Keys Sensitive -API keys are sensitive credentials that provide access to your Atlas organization and projects. They should be treated with the same level of security as passwords or other authentication tokens. +API keys are sensitive credentials that provide access to your Stratix organization and projects. They should be treated with the same level of security as passwords or other authentication tokens. **Risks of compromised API keys**: @@ -32,11 +32,11 @@ API keys are sensitive credentials that provide access to your Atlas organizatio ```python import os -from layerlens import Atlas +from layerlens import Stratix # Secure: Load from environment variables -client = Atlas( - api_key=os.getenv('LAYERLENS_ATLAS_API_KEY'), +client = Stratix( + api_key=os.getenv('LAYERLENS_STRATIX_API_KEY'), ) ``` @@ -46,7 +46,7 @@ client = Atlas( ```bash # Add to your shell profile (.bashrc, .zshrc, etc.) -export LAYERLENS_ATLAS_API_KEY="sk-your-key-here" +export LAYERLENS_STRATIX_API_KEY="sk-your-key-here" # Reload your shell configuration source ~/.bashrc # or ~/.zshrc @@ -56,10 +56,10 @@ source ~/.bashrc # or ~/.zshrc ```cmd # Command Prompt (persistent) -setx LAYERLENS_ATLAS_API_KEY "sk-your-key-here" +setx LAYERLENS_STRATIX_API_KEY "sk-your-key-here" # PowerShell (session-only) -$env:LAYERLENS_ATLAS_API_KEY="sk-your-key-here" +$env:LAYERLENS_STRATIX_API_KEY="sk-your-key-here" ``` ### Using .env Files @@ -68,7 +68,7 @@ $env:LAYERLENS_ATLAS_API_KEY="sk-your-key-here" ```bash # .env -LAYERLENS_ATLAS_API_KEY=sk-your-key-here +LAYERLENS_STRATIX_API_KEY=sk-your-key-here ``` **Load .env file in Python**: @@ -80,10 +80,10 @@ import os # Load environment variables from .env file load_dotenv() -from layerlens import Atlas +from layerlens import Stratix # Now environment variables are available -client = Atlas() +client = Stratix() ``` **Important**: Add `.env` to your `.gitignore` file: diff --git a/docs/security/environment-variables.md b/docs/security/environment-variables.md index 2830e42..9dc6645 100644 --- a/docs/security/environment-variables.md +++ b/docs/security/environment-variables.md @@ -1,18 +1,18 @@ # Environment Variables -This guide covers secure practices for managing environment variables when using the Atlas Python SDK. +This guide covers secure practices for managing environment variables when using the Stratix Python SDK. ## Overview -Environment variables provide a secure way to configure your Atlas SDK without hardcoding sensitive credentials in your source code. This approach separates configuration from code and enables different configurations for different environments. +Environment variables provide a secure way to configure your Stratix SDK without hardcoding sensitive credentials in your source code. This approach separates configuration from code and enables different configurations for different environments. ## Required Environment Variables -The Atlas SDK uses these primary environment variables: +The Stratix SDK uses these primary environment variables: | Variable | Description | Required | Example | | ------------------------- | ------------------ | -------- | -------------- | -| `LAYERLENS_ATLAS_API_KEY` | Your Atlas API key | Yes | `sk-abc123...` | +| `LAYERLENS_STRATIX_API_KEY` | Your Stratix API key | Yes | `sk-abc123...` | ## Setting Environment Variables @@ -22,10 +22,10 @@ The Atlas SDK uses these primary environment variables: ```bash # Set for current session -export LAYERLENS_ATLAS_API_KEY="sk-your-key-here" +export LAYERLENS_STRATIX_API_KEY="sk-your-key-here" # Add to shell profile for persistence (.bashrc, .zshrc, etc.) -echo 'export LAYERLENS_ATLAS_API_KEY="sk-your-key-here"' >> ~/.bashrc +echo 'export LAYERLENS_STRATIX_API_KEY="sk-your-key-here"' >> ~/.bashrc # Reload shell configuration source ~/.bashrc @@ -35,20 +35,20 @@ source ~/.bashrc ```cmd # Set for current session -set LAYERLENS_ATLAS_API_KEY=sk-your-key-here +set LAYERLENS_STRATIX_API_KEY=sk-your-key-here # Set permanently (requires admin rights) -setx LAYERLENS_ATLAS_API_KEY "sk-your-key-here" +setx LAYERLENS_STRATIX_API_KEY "sk-your-key-here" ``` **Windows PowerShell**: ```powershell # Set for current session -$env:LAYERLENS_ATLAS_API_KEY="sk-your-key-here" +$env:LAYERLENS_STRATIX_API_KEY="sk-your-key-here" # Set permanently for current user -[Environment]::SetEnvironmentVariable("LAYERLENS_ATLAS_API_KEY", "sk-your-key-here", "User") +[Environment]::SetEnvironmentVariable("LAYERLENS_STRATIX_API_KEY", "sk-your-key-here", "User") ``` ### Verification @@ -58,13 +58,13 @@ $env:LAYERLENS_ATLAS_API_KEY="sk-your-key-here" ```python import os -def verify_atlas_environment(): - """Verify Atlas environment variables are configured""" +def verify_stratix_environment(): + """Verify Stratix environment variables are configured""" required_vars = { - 'LAYERLENS_ATLAS_API_KEY': 'API Key', + 'LAYERLENS_STRATIX_API_KEY': 'API Key', } - print("🔍 Atlas Environment Variable Check") + print("🔍 Stratix Environment Variable Check") print("=" * 40) all_set = True @@ -88,7 +88,7 @@ def verify_atlas_environment(): return all_set # Run verification -verify_atlas_environment() +verify_stratix_environment() ``` ## Using .env Files @@ -99,10 +99,10 @@ verify_atlas_environment() ```bash # .env -LAYERLENS_ATLAS_API_KEY=sk-development-key-here +LAYERLENS_STRATIX_API_KEY=sk-development-key-here # Optional: Set environment name -ATLAS_ENV=development +STRATIX_ENV=development ``` **Loading .env files in Python**: @@ -121,11 +121,11 @@ load_dotenv('.env.development') load_dotenv('/path/to/your/.env') # Verify variables are loaded -from layerlens import Atlas +from layerlens import Stratix try: - client = Atlas() # Will use environment variables - print("✅ Atlas client initialized successfully") + client = Stratix() # Will use environment variables + print("✅ Stratix client initialized successfully") except Exception as e: print(f"❌ Failed to initialize client: {e}") ``` diff --git a/docs/security/rate-limiting.md b/docs/security/rate-limiting.md index 8761266..15eb35f 100644 --- a/docs/security/rate-limiting.md +++ b/docs/security/rate-limiting.md @@ -1,6 +1,6 @@ # Rate Limiting -This guide covers how to handle rate limiting when using the Atlas Python SDK, including best practices for avoiding rate limits and properly handling rate limit errors. +This guide covers how to handle rate limiting when using the Stratix Python SDK, including best practices for avoiding rate limits and properly handling rate limit errors. ## Identifying Rate Limit Errors @@ -10,10 +10,10 @@ When you exceed rate limits, the API returns a `429 Too Many Requests` status: ```python import layerlens -from layerlens import Atlas +from layerlens import Stratix try: - client = Atlas() + client = Stratix() # Making too many requests quickly for i in range(100): diff --git a/docs/troubleshooting/authentication.md b/docs/troubleshooting/authentication.md index 007dd5b..d7574b7 100644 --- a/docs/troubleshooting/authentication.md +++ b/docs/troubleshooting/authentication.md @@ -16,4 +16,4 @@ This guide covers common authentication-related issues when using the Layerlens ### Missing Required Configuration -**Error**: `AtlasError: The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable` \ No newline at end of file +**Error**: `StratixError: The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_STRATIX_API_KEY environment variable` \ No newline at end of file diff --git a/docs/troubleshooting/error-codes.md b/docs/troubleshooting/error-codes.md index 38a0a2e..269e507 100644 --- a/docs/troubleshooting/error-codes.md +++ b/docs/troubleshooting/error-codes.md @@ -5,7 +5,7 @@ This reference guide provides detailed information about all error codes and exc ## Exception Hierarchy ``` -AtlasError (Base exception) +StratixError (Base exception) ├── APIError (Base for API-related errors) │ ├── APIConnectionError (Network/connection issues) │ │ └── APITimeoutError (Request timeouts) diff --git a/examples/all_results_no_pagination.py b/examples/all_results_no_pagination.py index 3a91e6b..173828a 100644 --- a/examples/all_results_no_pagination.py +++ b/examples/all_results_no_pagination.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get() diff --git a/examples/async_client.py b/examples/async_client.py index b2d6fb1..c64c2de 100644 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get(type="public", name="gpt-4o") diff --git a/examples/async_client_simple.py b/examples/async_client_simple.py index ed2a94e..1c4623e 100644 --- a/examples/async_client_simple.py +++ b/examples/async_client_simple.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get() diff --git a/examples/async_judges_and_traces.py b/examples/async_judges_and_traces.py new file mode 100644 index 0000000..de78e0e --- /dev/null +++ b/examples/async_judges_and_traces.py @@ -0,0 +1,82 @@ +#!/usr/bin/env -S poetry run python + +import os +import time +import asyncio + +# Also import sync client just to fetch a model (models resource works the same) +from layerlens import Stratix, AsyncStratix + + +async def main(): + # Fetch a model to use for judge creation (using sync client for simplicity) + sync_client = Stratix() + models = sync_client.models.get(type="public", name="gpt-4o") + if not models: + print("No models found, exiting") + return + model = models[0] + print(f"Using model: {model.name} ({model.id})") + + # Construct async client + client = AsyncStratix() + + # --- Create a judge + judge = await client.judges.create( + name=f"Response Quality Judge {int(time.time())}", + evaluation_goal="Evaluate whether the response is accurate, helpful, and well-structured", + model_id=model.id, + ) + print(f"Created judge {judge.id}: {judge.name}") + + # --- Upload traces + traces_file = os.path.join(os.path.dirname(__file__), "traces.jsonl") + result = await client.traces.upload(traces_file) + print(f"Uploaded {len(result.trace_ids)} traces") + + # --- List traces + traces_response = await client.traces.get_many(page_size=10) + print(f"Found {traces_response.total_count} traces") + + # --- Run judge on multiple traces concurrently + trace_ids = [t.id for t in traces_response.traces[:5]] + + # Estimate cost first + estimate = await client.trace_evaluations.estimate_cost( + trace_ids=trace_ids, + judge_id=judge.id, + ) + print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + + # Run evaluations concurrently + tasks = [client.trace_evaluations.create(trace_id=tid, judge_id=judge.id) for tid in trace_ids] + evaluations = await asyncio.gather(*tasks) + + for evaluation in evaluations: + if evaluation: + print(f" Evaluation {evaluation.id}: {evaluation.status}") + + # --- Wait for evaluations to finish, then fetch results + print("Waiting for evaluations to complete...") + await asyncio.sleep(10) + + for evaluation in evaluations: + if not evaluation: + continue + try: + results_response = await client.trace_evaluations.get_results(evaluation.id) + if results_response and results_response.results: + for result in results_response.results: + print(f" Score: {result.score}, Passed: {result.passed}") + else: + print(f" Evaluation {evaluation.id}: no results yet") + except Exception: + print(f" Evaluation {evaluation.id}: results not available yet") + + # --- Clean up + await client.judges.delete(judge.id) + print(f"Cleaned up judge {judge.id}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/async_run_evaluations.py b/examples/async_run_evaluations.py index c997e2f..3764e17 100644 --- a/examples/async_run_evaluations.py +++ b/examples/async_run_evaluations.py @@ -2,7 +2,7 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def create_and_run_evaluation(client, model, benchmark, eval_number): @@ -38,7 +38,7 @@ async def create_and_run_evaluation(client, model, benchmark, eval_number): async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get() diff --git a/examples/client.py b/examples/client.py index 58b6ee7..8047cea 100644 --- a/examples/client.py +++ b/examples/client.py @@ -1,9 +1,9 @@ #!/usr/bin/env -S poetry run python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client (API key from env or inline) -client = Atlas() +client = Stratix() # --- Models models = client.models.get() diff --git a/examples/client_simple.py b/examples/client_simple.py index 28b264b..7f1c7c6 100644 --- a/examples/client_simple.py +++ b/examples/client_simple.py @@ -1,9 +1,9 @@ #!/usr/bin/env -S poetry run python -from layerlens import Atlas +from layerlens import Stratix # Construct sync client (API key from env or inline) -client = Atlas() +client = Stratix() # --- Models models = client.models.get(type="public", name="gpt-4o") diff --git a/examples/fetch_results_async.py b/examples/fetch_results_async.py index 5d276c2..d9ef929 100644 --- a/examples/fetch_results_async.py +++ b/examples/fetch_results_async.py @@ -2,7 +2,7 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def fetch_evaluation_results(client, evaluation_id): @@ -26,7 +26,7 @@ async def fetch_evaluation_results(client, evaluation_id): async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # List of evaluation IDs to fetch exmple diff --git a/examples/get_benchmarks.py b/examples/get_benchmarks.py index 344d89b..169ffa9 100644 --- a/examples/get_benchmarks.py +++ b/examples/get_benchmarks.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Get benchmarks by name benchmark_name = "mmlu" diff --git a/examples/get_evaluation.py b/examples/get_evaluation.py index 0d841fc..45d667c 100644 --- a/examples/get_evaluation.py +++ b/examples/get_evaluation.py @@ -2,16 +2,16 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Get evaluation by id evaluation_id = "eval_123" - evaluation = await client.evaluations.get(evaluation_id) + evaluation = await client.evaluations.get_by_id(evaluation_id) print(f"Found evaluation {evaluation.id}") print(evaluation) diff --git a/examples/get_models.py b/examples/get_models.py index a4d9d1c..c8d14de 100644 --- a/examples/get_models.py +++ b/examples/get_models.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Get models by name model_name = "gpt-4o" diff --git a/examples/judge_optimizations.py b/examples/judge_optimizations.py new file mode 100644 index 0000000..9681ec9 --- /dev/null +++ b/examples/judge_optimizations.py @@ -0,0 +1,113 @@ +#!/usr/bin/env -S poetry run python + +""" +Judge Optimizations example. + +Note: Optimization requires that the judge has at least 10 annotations +(trace evaluation results). Run trace evaluations first to build up +annotation data before attempting optimization. +""" + +import time + +import layerlens +from layerlens import Stratix + +# Construct sync client (API key from env or inline) +client = Stratix() + +# --- Fetch a model to use for judge creation +models = client.models.get(type="public", name="gpt-4o") +if not models: + print("No models found, exiting") + exit(1) +model = models[0] +print(f"Using model: {model.name} ({model.id})") + +# --- Create a judge to optimize +judge = client.judges.create( + name=f"Optimization Demo Judge {int(time.time())}", + evaluation_goal="Evaluate whether the response is accurate, complete, and well-structured", + model_id=model.id, +) +print(f"Created judge {judge.id}: {judge.name}") + +# --- Estimate the cost of optimization +estimate = client.judge_optimizations.estimate( + judge_id=judge.id, + budget="medium", +) +if estimate: + print(f"Estimated cost: ${estimate.estimated_cost:.4f}") + print(f" Annotations: {estimate.annotation_count}") + print(f" Budget: {estimate.budget}") +else: + print("Could not estimate cost") + +# --- Create an optimization run +# Requires at least 10 annotations on the judge. +# If the judge doesn't have enough annotations, the API returns a 400 error. +try: + run = client.judge_optimizations.create( + judge_id=judge.id, + budget="medium", + ) +except layerlens.BadRequestError as e: + print(f"Cannot start optimization: {e}") + print("Tip: Run trace evaluations with this judge first to build up annotations.") + # Demonstrate list and clean up even without a successful run + response = client.judge_optimizations.get_many(judge_id=judge.id) + if response: + print(f"Found {response.total} optimization runs for this judge") + client.judges.delete(judge.id) + print(f"Cleaned up judge {judge.id}") + exit(0) + +if not run: + print("Failed to create optimization run") + client.judges.delete(judge.id) + exit(1) +print(f"Created optimization run {run.id}, status: {run.status}") + +# --- Poll for completion (optimization can take a while) +optimization = None +print("Waiting for optimization to complete...") +for i in range(60): + optimization = client.judge_optimizations.get(run.id) + if not optimization: + print(" Could not fetch optimization run") + break + print(f" [{i * 5}s] Status: {optimization.status}") + if optimization.status.value in ("success", "failure"): + print(f" Baseline accuracy: {optimization.baseline_accuracy}") + print(f" Optimized accuracy: {optimization.optimized_accuracy}") + if optimization.original_goal: + print(f" Original goal: {optimization.original_goal[:80]}...") + if optimization.optimized_goal: + print(f" Optimized goal: {optimization.optimized_goal[:80]}...") + print(f" Actual cost: ${optimization.actual_cost:.4f}") + break + time.sleep(5) + +# --- List optimization runs for this judge +response = client.judge_optimizations.get_many(judge_id=judge.id) +if response: + print(f"Found {response.total} optimization runs for this judge") + for r in response.optimization_runs: + print(f" - {r.id}: {r.status} (budget: {r.budget})") + +# --- Apply optimization results (only if optimization succeeded) +if optimization and optimization.status.value == "success": + result = client.judge_optimizations.apply(run.id) + if result: + print(f"Applied optimization to judge {result.judge_id}") + print(f" New version: v{result.new_version}") + print(f" {result.message}") + else: + print("Could not apply optimization result") +else: + print("Skipping apply (optimization did not succeed)") + +# --- Clean up +client.judges.delete(judge.id) +print(f"Cleaned up judge {judge.id}") diff --git a/examples/judges.py b/examples/judges.py new file mode 100644 index 0000000..b6a3e3e --- /dev/null +++ b/examples/judges.py @@ -0,0 +1,46 @@ +#!/usr/bin/env -S poetry run python + +import time + +from layerlens import Stratix + +# Construct sync client (API key from env or inline) +client = Stratix() + +# --- Fetch a model to use as the judge's LLM +models = client.models.get(type="public", name="gpt-4o") +if not models: + print("No models found, exiting") + exit(1) +model = models[0] +print(f"Using model: {model.name} ({model.id})") + +# --- Create a judge +judge = client.judges.create( + name=f"Code Quality Judge {int(time.time())}", + evaluation_goal="Evaluate the quality of code output including correctness, readability, and style", + model_id=model.id, +) +print(f"Created judge {judge.id}: {judge.name}") + +# --- Get a judge by ID +judge = client.judges.get(judge.id) +print(f"Judge: {judge.name}, version: {judge.version}") + +# --- List all judges +response = client.judges.get_many() +print(f"Found {response.total_count} judges") +for j in response.judges: + print(f" - {j.name} (v{j.version}, {j.run_count} runs)") + +# --- Update a judge +updated = client.judges.update( + judge.id, + name="Updated Code Quality Judge", + evaluation_goal="Evaluate code output for correctness, readability, style, and security", +) +print(f"Updated judge {updated.id}") + +# --- Delete a judge +deleted = client.judges.delete(judge.id) +print(f"Deleted judge {deleted.id}") diff --git a/examples/paginated_results.py b/examples/paginated_results.py index 7be00d7..2f135e7 100644 --- a/examples/paginated_results.py +++ b/examples/paginated_results.py @@ -2,12 +2,12 @@ import asyncio -from layerlens import AsyncAtlas +from layerlens import AsyncStratix async def main(): # Construct async client - client = AsyncAtlas() + client = AsyncStratix() # --- Models models = await client.models.get() diff --git a/examples/trace_evaluations.py b/examples/trace_evaluations.py new file mode 100644 index 0000000..e956598 --- /dev/null +++ b/examples/trace_evaluations.py @@ -0,0 +1,80 @@ +#!/usr/bin/env -S poetry run python + +import time + +from layerlens import Stratix + +# Construct sync client (API key from env or inline) +client = Stratix() + +# --- Fetch a model to use for judge creation +models = client.models.get(type="public", name="gpt-4o") +if not models: + print("No models found, exiting") + exit(1) +model = models[0] +print(f"Using model: {model.name} ({model.id})") + +# --- Create a judge to use for evaluations +judge = client.judges.create( + name=f"Trace Eval Demo Judge {int(time.time())}", + evaluation_goal="Evaluate whether the response is accurate, complete, and well-structured", + model_id=model.id, +) +print(f"Created judge {judge.id}: {judge.name}") + +# --- Get existing traces to evaluate +traces_response = client.traces.get_many(page_size=3) +if not traces_response or len(traces_response.traces) == 0: + print("No traces found. Upload some traces first using traces.py") + # Clean up the judge + client.judges.delete(judge.id) + exit(1) + +trace_ids = [t.id for t in traces_response.traces] +print(f"Found {len(trace_ids)} traces to evaluate") + +# --- Estimate cost before running +estimate = client.trace_evaluations.estimate_cost( + trace_ids=trace_ids, + judge_id=judge.id, +) +print(f"Estimated cost: ${estimate.estimated_cost:.4f} for {estimate.trace_count} traces") + +# --- Run a judge on the first trace +evaluation = client.trace_evaluations.create( + trace_id=trace_ids[0], + judge_id=judge.id, +) +print(f"Created evaluation {evaluation.id}, status: {evaluation.status}") + +# --- Wait for evaluation to complete (poll every 2 seconds, up to 60s) +for _ in range(30): + evaluation = client.trace_evaluations.get(evaluation.id) + print(f"Evaluation status: {evaluation.status}") + if evaluation.status.value in ("success", "failure"): + break + time.sleep(2) + +# --- Get evaluation results (may 404 if still in progress) +try: + results_response = client.trace_evaluations.get_results(evaluation.id) + if results_response and results_response.results: + for result in results_response.results: + print(f" Score: {result.score}, Passed: {result.passed}") + print(f" Reasoning: {result.reasoning}") + if result.steps: + for step in result.steps: + print(f" Step {step.step}: {step.reasoning}") + else: + print(" No results returned") +except Exception: + print(" No results yet (evaluation may still be in progress)") + +# --- List all trace evaluations +response = client.trace_evaluations.get_many() +print(f"Found {response.total} trace evaluations") + +# --- Clean up +client.judges.delete(judge.id) +print(f"Cleaned up judge {judge.id}") diff --git a/examples/traces.jsonl b/examples/traces.jsonl new file mode 100644 index 0000000..dedbea4 --- /dev/null +++ b/examples/traces.jsonl @@ -0,0 +1,3 @@ +{"input": "What is the capital of France?", "output": "The capital of France is Paris.", "model": "gpt-4", "latency_ms": 450, "tokens": 28} +{"input": "Explain quantum computing in simple terms.", "output": "Quantum computing uses quantum bits (qubits) that can exist in multiple states simultaneously, unlike classical bits which are either 0 or 1. This allows quantum computers to process many possibilities at once, making them potentially much faster for certain types of problems like cryptography and drug discovery.", "model": "gpt-4", "latency_ms": 1200, "tokens": 65} +{"input": "Write a Python function to reverse a string.", "output": "def reverse_string(s):\n return s[::-1]", "model": "gpt-4", "latency_ms": 320, "tokens": 22} diff --git a/examples/traces.py b/examples/traces.py new file mode 100644 index 0000000..af99970 --- /dev/null +++ b/examples/traces.py @@ -0,0 +1,39 @@ +#!/usr/bin/env -S poetry run python + +import os + +from layerlens import Stratix + +# Construct sync client (API key from env or inline) +client = Stratix() + +# --- Upload traces from a file +traces_file = os.path.join(os.path.dirname(__file__), "traces.jsonl") +result = client.traces.upload(traces_file) +print(f"Uploaded {len(result.trace_ids)} traces") + +# --- List traces +response = client.traces.get_many() +print(f"Found {response.total_count} traces") +for trace in response.traces[:5]: + print(f" - {trace.id}: {trace.filename}") + +# --- List traces with filters +filtered = client.traces.get_many( + sort_by="created_at", + sort_order="desc", + page_size=10, +) +print(f"Filtered traces: {filtered.count}") + +# --- Get a single trace (use the uploaded trace ID) +trace = client.traces.get(result.trace_ids[0]) +print(f"Trace {trace.id}: {len(trace.data)} data keys") + +# --- Get available sources +sources = client.traces.get_sources() +print(f"Sources: {sources}") + +# --- Delete a trace +deleted = client.traces.delete(trace.id) +print(f"Deleted: {deleted}") diff --git a/package-lock.json b/package-lock.json index 9aadb01..367f32b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "atlas-python", + "name": "stratix-python", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/pyproject.toml b/pyproject.toml index b5ec9a9..f5a057f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "layerlens" -version = "1.0.2" -description = "The official Python library for the LayerLens Atlas API" +version = "1.1.0" +description = "The official Python library for the LayerLens Stratix API" license = "Apache-2.0" authors = [{ name = "LayerLens", email = "support@layerlens.ai" }] dependencies = ["httpx>=0.23.0, <1", "pydantic>=1.9.0, <3", "requests"] @@ -24,8 +24,8 @@ classifiers = [ ] [project.urls] -Homepage = "https://github.com/LayerLens/atlas-python" -Repository = "https://github.com/LayerLens/atlas-python" +Homepage = "https://github.com/LayerLens/stratix-python" +Repository = "https://github.com/LayerLens/stratix-python" [project.scripts] layerlens = "layerlens.cli:main" diff --git a/scripts/push_release_tag.sh b/scripts/push_release_tag.sh index e6529ce..fe6ca6e 100644 --- a/scripts/push_release_tag.sh +++ b/scripts/push_release_tag.sh @@ -22,7 +22,7 @@ done git fetch --tags --prune -REPO_URL="https://github.com/LayerLens/atlas-python" +REPO_URL="https://github.com/LayerLens/stratix-python" # e.g. v1.0.0, v1.0.1, etc. TAG_PREFIX="v" COMMIT=$(git rev-parse --short HEAD) @@ -38,7 +38,7 @@ fi LAST_RELEASE=$(git tag -l "${TAG_PREFIX}*" --sort=-v:refname | head -n 1) echo "================================================" -echo " Atlas Python SDK Release" +echo " Stratix Python SDK Release" echo "================================================" echo "version: ${TAG}" echo "commit: ${COMMIT}" diff --git a/src/CONTRIBUTING.md b/src/CONTRIBUTING.md index fb73d10..f352d82 100644 --- a/src/CONTRIBUTING.md +++ b/src/CONTRIBUTING.md @@ -62,7 +62,7 @@ If you’d like to use the repository from source, you can either install from g To install via git: ```sh -$ pip install git+ssh://git@github.com/LayerLens/atlas-python.git +$ pip install git+ssh://git@github.com/LayerLens/stratix-python.git ``` Alternatively, you can build from source and install the wheel file: @@ -109,8 +109,7 @@ $ ./scripts/format ## Release Process -This section outlines the complete process for releasing a new version of the Atlas Python SDK. - +This section outlines the complete process for releasing a new version of the Stratix Python SDK. ### Step-by-Step Release Process @@ -133,22 +132,24 @@ $ git rebase main Update the version number in the following files: -- **`src/atlas/_version.py`**: Update the `__version__` string +- **`src/layerlens/_version.py`**: Update the `__version__` string - **`pyproject.toml`**: Update the `version` field **Version Format**: Use semantic versioning (MAJOR.MINOR.PATCH) + - **MAJOR**: Breaking changes - **MINOR**: New features (backward compatible) - **PATCH**: Bug fixes (backward compatible) **Examples**: + - `1.0.2` → `1.0.3` (patch release for bug fixes) - `1.0.2` → `1.1.0` (minor release for new features) - `1.0.2` → `2.0.0` (major release for breaking changes) ```sh # Example: Update to version 1.0.3 -# Edit src/atlas/_version.py +# Edit src/layerlens/_version.py __version__ = "1.0.3" # Edit pyproject.toml @@ -172,7 +173,7 @@ $ rye build ```sh # Add and commit the version bump -$ git add src/atlas/_version.py pyproject.toml +$ git add src/layerlens/_version.py pyproject.toml $ git commit -m "chore: bump version to 1.0.3" # Push the release branch @@ -225,7 +226,7 @@ Use this checklist to ensure you we don't miss any steps: - [ ] Switched to and updated the `release` branch - [ ] Merged latest changes from `main` -- [ ] Updated version in `src/atlas/_version.py` +- [ ] Updated version in `src/layerlens/_version.py` - [ ] Updated version in `pyproject.toml` - [ ] Ran tests (`./scripts/test`) - [ ] Ran linting (`./scripts/lint`) @@ -238,6 +239,7 @@ Use this checklist to ensure you we don't miss any steps: ### Troubleshooting **Tag already exists**: If you need to recreate a tag, delete it first: + ```sh $ git tag -d v1.0.3 # Delete locally $ git push origin :v1.0.3 # Delete on remote diff --git a/src/README.md b/src/README.md index 2ae26e0..e8ad0c6 100644 --- a/src/README.md +++ b/src/README.md @@ -1,23 +1,23 @@ -# Atlas Python API library +# Stratix Python API library -The Atlas Python library provides convenient access to the Atlas REST API from any Python 3.8+ application. The library offers synchronous clients powered by [httpx](https://github.com/encode/httpx). +The Stratix Python library provides convenient access to the Stratix REST API from any Python 3.8+ application. The library offers synchronous clients powered by [httpx](https://github.com/encode/httpx). ## Installation ```sh # Install from PyPI -pip install atlas --index-url https://sdk.layerlens.ai +pip install stratix --index-url https://sdk.layerlens.ai ``` ## Usage ```python import os -from layerlens import Atlas +from layerlens import Stratix -client = Atlas( +client = Stratix( # This is the default and can be omitted - api_key=os.environ.get("LAYERLENS_ATLAS_API_KEY"), + api_key=os.environ.get("LAYERLENS_STRATIX_API_KEY"), ) evaluation = client.evaluations.create( @@ -38,9 +38,9 @@ All errors inherit from `layerlens.APIError`. ```python import layerlens -from layerlens import Atlas +from layerlens import Stratix -client = Atlas() +client = Stratix() try: client.evaluations.create( @@ -76,16 +76,16 @@ Error codes are as follows: By default requests time out after 10 minutes. You can configure this with a `timeout` option, which accepts a float or an [`httpx.Timeout`](https://www.python-httpx.org/advanced/timeouts/#fine-tuning-the-configuration) object: ```python -from layerlens import Atlas +from layerlens import Stratix # Configure the default for all requests: -client = Atlas( +client = Stratix( # 20 seconds (default is 10 minutes) timeout=20.0, ) # More granular control: -client = Atlas( +client = Stratix( timeout=httpx.Timeout(60.0, read=5.0, write=10.0, connect=2.0), ) diff --git a/src/layerlens/__init__.py b/src/layerlens/__init__.py index c229409..a5fcc80 100644 --- a/src/layerlens/__init__.py +++ b/src/layerlens/__init__.py @@ -1,3 +1,52 @@ -from ._client import Atlas, Client, AsyncAtlas, AsyncClient +from .models import ( + Judge, + Trace, + JudgeVersion, + JudgeSnapshot, + TraceEvaluation, + OptimizationBudget, + TraceEvaluationStep, + JudgeOptimizationRun, + TraceWithEvaluations, + OptimizationRunStatus, + TraceEvaluationResult, + TraceEvaluationStatus, + TraceEvaluationSummary, +) +from ._client import Atlas, Client, Stratix, AsyncAtlas, AsyncClient, AsyncStratix +from ._exceptions import ( + APIError, + AtlasError, + StratixError, + NotFoundError, + BadRequestError, + AuthenticationError, +) -__all__ = ["Atlas", "AsyncAtlas", "Client", "AsyncClient"] +__all__ = [ + "APIError", + "AsyncAtlas", + "AsyncClient", + "AsyncStratix", + "Atlas", + "AtlasError", + "AuthenticationError", + "BadRequestError", + "Client", + "Judge", + "JudgeOptimizationRun", + "JudgeSnapshot", + "JudgeVersion", + "NotFoundError", + "OptimizationBudget", + "OptimizationRunStatus", + "Stratix", + "StratixError", + "Trace", + "TraceEvaluation", + "TraceEvaluationResult", + "TraceEvaluationStatus", + "TraceEvaluationStep", + "TraceEvaluationSummary", + "TraceWithEvaluations", +] diff --git a/src/layerlens/_base_client.py b/src/layerlens/_base_client.py index 8af1bb3..6fbf3d1 100644 --- a/src/layerlens/_base_client.py +++ b/src/layerlens/_base_client.py @@ -96,6 +96,28 @@ def post_cast( ) -> Union[ResponseT, httpx.Response]: return self._request_cast("POST", url, cast_to=cast_to, body=body, headers=headers, **kwargs) + def patch_cast( + self, + url: str, + *, + cast_to: Optional[Type[ResponseT]] = None, + body: Optional[Any] = None, + headers: Optional[Dict[str, str]] = None, + **kwargs: Any, + ) -> Union[ResponseT, httpx.Response]: + return self._request_cast("PATCH", url, cast_to=cast_to, body=body, headers=headers, **kwargs) + + def delete_cast( + self, + url: str, + *, + cast_to: Optional[Type[ResponseT]] = None, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + **kwargs: Any, + ) -> Union[ResponseT, httpx.Response]: + return self._request_cast("DELETE", url, cast_to=cast_to, params=params, headers=headers, **kwargs) + def _make_status_error_from_response( self, response: httpx.Response, @@ -200,6 +222,28 @@ async def post_cast( ) -> Union[ResponseT, httpx.Response]: return await self._request_cast("POST", url, cast_to=cast_to, body=body, headers=headers, **kwargs) + async def patch_cast( + self, + url: str, + *, + cast_to: Optional[Type[ResponseT]] = None, + body: Optional[Any] = None, + headers: Optional[Dict[str, str]] = None, + **kwargs: Any, + ) -> Union[ResponseT, httpx.Response]: + return await self._request_cast("PATCH", url, cast_to=cast_to, body=body, headers=headers, **kwargs) + + async def delete_cast( + self, + url: str, + *, + cast_to: Optional[Type[ResponseT]] = None, + params: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + **kwargs: Any, + ) -> Union[ResponseT, httpx.Response]: + return await self._request_cast("DELETE", url, cast_to=cast_to, params=params, headers=headers, **kwargs) + def _make_status_error_from_response( self, response: httpx.Response, diff --git a/src/layerlens/_client.py b/src/layerlens/_client.py index a1c2a6b..9599ab9 100644 --- a/src/layerlens/_client.py +++ b/src/layerlens/_client.py @@ -13,20 +13,24 @@ from ._utils import is_mapping from .models import Organization, OrganizationResponse from ._constants import DEFAULT_TIMEOUT -from ._exceptions import AtlasError, APIStatusError +from ._exceptions import StratixError, APIStatusError from ._base_client import BaseClient, BaseAsyncClient if TYPE_CHECKING: + from .resources.judges import Judges, AsyncJudges from .resources.models import Models, AsyncModels + from .resources.traces import Traces, AsyncTraces from .resources.results import Results, AsyncResults from .resources.benchmarks import Benchmarks, AsyncBenchmarks from .resources.evaluations import Evaluations, AsyncEvaluations + from .resources.trace_evaluations import TraceEvaluations, AsyncTraceEvaluations + from .resources.judge_optimizations import JudgeOptimizations, AsyncJudgeOptimizations -__all__ = ["Atlas", "Client"] +__all__ = ["Stratix", "Client"] -class Atlas(BaseClient): +class Stratix(BaseClient): api_key: str organization_id: str | None project_id: str | None @@ -38,21 +42,21 @@ def __init__( base_url: str | httpx.URL | None = None, timeout: Union[float, httpx.Timeout, None] = DEFAULT_TIMEOUT, ) -> None: - """Construct a new synchronous Atlas client instance. + """Construct a new synchronous Stratix client instance. This automatically infers the following arguments from their corresponding environment variables if they are not provided: - - `api_key` from `LAYERLENS_ATLAS_API_KEY` + - `api_key` from `LAYERLENS_STRATIX_API_KEY` """ if api_key is None: - api_key = os.environ.get("LAYERLENS_ATLAS_API_KEY") + api_key = os.environ.get("LAYERLENS_STRATIX_API_KEY") or os.environ.get("LAYERLENS_ATLAS_API_KEY") if api_key is None or api_key == "": - raise AtlasError( - "The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable" + raise StratixError( + "The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_STRATIX_API_KEY environment variable" ) self.api_key = api_key if base_url is None: - base_url = os.environ.get("LAYERLENS_ATLAS_BASE_URL") + base_url = os.environ.get("LAYERLENS_STRATIX_BASE_URL") or os.environ.get("LAYERLENS_ATLAS_BASE_URL") if base_url is None: base_url = "https://api.layerlens.ai/api/v1" @@ -63,12 +67,12 @@ def __init__( organization = self._get_organization() if organization is None: - raise AtlasError(f"Organization could not be fetched. Please contact LayerLens Atlas support.") + raise StratixError(f"Organization could not be fetched. Please contact LayerLens Stratix support.") self.organization_id = organization.id if organization.projects is None or len(organization.projects) == 0: - raise AtlasError( - f"Organization {self.organization_id} is missing project. Please contact LayerLens Atlas support." + raise StratixError( + f"Organization {self.organization_id} is missing project. Please contact LayerLens Stratix support." ) self.project_id = organization.projects[0].id @@ -84,6 +88,18 @@ def evaluations(self) -> Evaluations: return Evaluations(self) + @cached_property + def judges(self) -> Judges: + from .resources.judges import Judges + + return Judges(self) + + @cached_property + def judge_optimizations(self) -> JudgeOptimizations: + from .resources.judge_optimizations import JudgeOptimizations + + return JudgeOptimizations(self) + @cached_property def models(self) -> Models: from .resources.models import Models @@ -96,6 +112,18 @@ def results(self) -> Results: return Results(self) + @cached_property + def traces(self) -> Traces: + from .resources.traces import Traces + + return Traces(self) + + @cached_property + def trace_evaluations(self) -> TraceEvaluations: + from .resources.trace_evaluations import TraceEvaluations + + return TraceEvaluations(self) + @property @override def auth_headers(self) -> dict[str, str]: @@ -167,7 +195,7 @@ def _get_organization(self) -> Optional[Organization]: return organization.data if isinstance(organization, OrganizationResponse) else None -class AsyncAtlas(BaseAsyncClient): +class AsyncStratix(BaseAsyncClient): api_key: str organization_id: str | None project_id: str | None @@ -179,22 +207,22 @@ def __init__( base_url: str | httpx.URL | None = None, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> None: - """Construct a new asynchronous Atlas client instance. + """Construct a new asynchronous Stratix client instance. This automatically infers the following arguments from their corresponding environment variables if they are not provided: - - `api_key` from `LAYERLENS_ATLAS_API_KEY` + - `api_key` from `LAYERLENS_STRATIX_API_KEY` """ if api_key is None: - api_key = os.environ.get("LAYERLENS_ATLAS_API_KEY") + api_key = os.environ.get("LAYERLENS_STRATIX_API_KEY") or os.environ.get("LAYERLENS_ATLAS_API_KEY") if api_key is None or api_key == "": - raise AtlasError( + raise StratixError( "The api_key client option must be set either by passing api_key to the client " - "or by setting the LAYERLENS_ATLAS_API_KEY environment variable" + "or by setting the LAYERLENS_STRATIX_API_KEY environment variable" ) self.api_key = api_key if base_url is None: - base_url = os.environ.get("LAYERLENS_ATLAS_BASE_URL") + base_url = os.environ.get("LAYERLENS_STRATIX_BASE_URL") or os.environ.get("LAYERLENS_ATLAS_BASE_URL") if base_url is None: base_url = "https://api.layerlens.ai/api/v1" @@ -202,12 +230,12 @@ def __init__( organization = self._get_organization() if organization is None: - raise AtlasError(f"Organization could not be fetched. Please contact LayerLens Atlas support.") + raise StratixError(f"Organization could not be fetched. Please contact LayerLens Stratix support.") self.organization_id = organization.id if organization.projects is None or len(organization.projects) == 0: - raise AtlasError( - f"Organization {self.organization_id} is missing project. Please contact LayerLens Atlas support." + raise StratixError( + f"Organization {self.organization_id} is missing project. Please contact LayerLens Stratix support." ) self.project_id = organization.projects[0].id @@ -223,6 +251,18 @@ def evaluations(self) -> AsyncEvaluations: return AsyncEvaluations(self) + @cached_property + def judges(self) -> AsyncJudges: + from .resources.judges import AsyncJudges + + return AsyncJudges(self) + + @cached_property + def judge_optimizations(self) -> AsyncJudgeOptimizations: + from .resources.judge_optimizations import AsyncJudgeOptimizations + + return AsyncJudgeOptimizations(self) + @cached_property def models(self) -> AsyncModels: from .resources.models import AsyncModels @@ -235,6 +275,18 @@ def results(self) -> AsyncResults: return AsyncResults(self) + @cached_property + def traces(self) -> AsyncTraces: + from .resources.traces import AsyncTraces + + return AsyncTraces(self) + + @cached_property + def trace_evaluations(self) -> AsyncTraceEvaluations: + from .resources.trace_evaluations import AsyncTraceEvaluations + + return AsyncTraceEvaluations(self) + @property @override def auth_headers(self) -> dict[str, str]: @@ -298,5 +350,9 @@ def _get_organization(self) -> Optional[Organization]: return organization.data if isinstance(organization, OrganizationResponse) else None -Client = Atlas -AsyncClient = AsyncAtlas +Client = Stratix +AsyncClient = AsyncStratix + +# Backward-compatibility aliases +Atlas = Stratix +AsyncAtlas = AsyncStratix diff --git a/src/layerlens/_exceptions.py b/src/layerlens/_exceptions.py index ee4306b..7a24169 100644 --- a/src/layerlens/_exceptions.py +++ b/src/layerlens/_exceptions.py @@ -6,11 +6,11 @@ import httpx -class AtlasError(Exception): +class StratixError(Exception): pass -class APIError(AtlasError): +class APIError(StratixError): message: str request: httpx.Request @@ -106,3 +106,7 @@ class RateLimitError(APIStatusError): class InternalServerError(APIStatusError): pass + + +# Backward-compatibility alias +AtlasError = StratixError diff --git a/src/layerlens/_resource.py b/src/layerlens/_resource.py index c9c8c06..973ae62 100644 --- a/src/layerlens/_resource.py +++ b/src/layerlens/_resource.py @@ -5,28 +5,32 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from ._client import Atlas, AsyncAtlas + from ._client import Stratix, AsyncStratix class SyncAPIResource: - _client: Atlas + _client: Stratix - def __init__(self, client: Atlas) -> None: + def __init__(self, client: Stratix) -> None: self._client = client self._get = client.get_cast self._post = client.post_cast + self._patch = client.patch_cast + self._delete = client.delete_cast def _sleep(self, seconds: float) -> None: time.sleep(seconds) class AsyncAPIResource: - _client: AsyncAtlas + _client: AsyncStratix - def __init__(self, client: AsyncAtlas) -> None: + def __init__(self, client: AsyncStratix) -> None: self._client = client self._get = client.get_cast self._post = client.post_cast + self._patch = client.patch_cast + self._delete = client.delete_cast async def _sleep(self, seconds: float) -> None: await asyncio.sleep(seconds) diff --git a/src/layerlens/_version.py b/src/layerlens/_version.py index 7863915..6849410 100644 --- a/src/layerlens/_version.py +++ b/src/layerlens/_version.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.1.0" diff --git a/src/layerlens/models/__init__.py b/src/layerlens/models/__init__.py index e793879..85c1da0 100644 --- a/src/layerlens/models/__init__.py +++ b/src/layerlens/models/__init__.py @@ -1,29 +1,72 @@ from .api import ( Pagination, ResultMetrics, + JudgesResponse, ModelsResponse, + TracesResponse, ResultsResponse, + UploadURLResponse, BenchmarksResponse, + CreateJudgeResponse, + DeleteJudgeResponse, EvaluationsResponse, + UpdateJudgeResponse, + CostEstimateResponse, + CreateTracesResponse, OrganizationResponse, + TraceEvaluationsResponse, CreateEvaluationsResponse, + JudgeOptimizationRunsResponse, + TraceEvaluationResultsResponse, + CreateJudgeOptimizationRunResponse, + ApplyJudgeOptimizationResultResponse, + EstimateJudgeOptimizationCostResponse, ) +from .judge import Judge, JudgeVersion from .model import Model, CustomModel, PublicModel +from .trace import Trace, TraceWithEvaluations, TraceEvaluationSummary from .benchmark import Benchmark, CustomBenchmark, PublicBenchmark from .evaluation import Result, Evaluation, EvaluationStatus from .organization import Project, Organization +from .trace_evaluation import ( + JudgeSnapshot, + TraceEvaluation, + TraceEvaluationStep, + TraceEvaluationResult, + TraceEvaluationStatus, +) +from .judge_optimization import ( + OptimizationBudget, + JudgeOptimizationRun, + OptimizationRunStatus, +) __all__ = [ + "ApplyJudgeOptimizationResultResponse", "Benchmark", "BenchmarksResponse", + "CostEstimateResponse", "CreateEvaluationsResponse", + "CreateJudgeOptimizationRunResponse", + "CreateJudgeResponse", + "CreateTracesResponse", "CustomBenchmark", "CustomModel", + "DeleteJudgeResponse", + "EstimateJudgeOptimizationCostResponse", "Evaluation", "EvaluationStatus", "EvaluationsResponse", + "Judge", + "JudgeOptimizationRun", + "JudgeOptimizationRunsResponse", + "JudgeSnapshot", + "JudgeVersion", + "JudgesResponse", "Model", "ModelsResponse", + "OptimizationBudget", + "OptimizationRunStatus", "Organization", "OrganizationResponse", "Pagination", @@ -33,4 +76,16 @@ "Result", "ResultMetrics", "ResultsResponse", + "Trace", + "TraceEvaluation", + "TraceEvaluationResult", + "TraceEvaluationResultsResponse", + "TraceEvaluationStatus", + "TraceEvaluationStep", + "TraceEvaluationsResponse", + "TraceEvaluationSummary", + "TraceWithEvaluations", + "TracesResponse", + "UpdateJudgeResponse", + "UploadURLResponse", ] diff --git a/src/layerlens/models/api.py b/src/layerlens/models/api.py index eba00db..398ebe4 100644 --- a/src/layerlens/models/api.py +++ b/src/layerlens/models/api.py @@ -4,10 +4,14 @@ from pydantic import Field, BaseModel, ConfigDict +from .judge import Judge from .model import Model +from .trace import TraceWithEvaluations from .benchmark import Benchmark from .evaluation import Result, Evaluation from .organization import Organization +from .trace_evaluation import TraceEvaluation, TraceEvaluationResult +from .judge_optimization import JudgeOptimizationRun class BenchmarksResponse(BaseModel): @@ -55,3 +59,84 @@ class ResultsResponse(BaseModel): results: List[Result] metrics: ResultMetrics pagination: Pagination + + +class JudgesResponse(BaseModel): + judges: List[Judge] + count: int + total_count: int + + +class CreateJudgeResponse(BaseModel): + id: str + + +class UpdateJudgeResponse(BaseModel): + organization_id: str + project_id: str + id: str + + +class DeleteJudgeResponse(BaseModel): + organization_id: str + project_id: str + id: str + + +class TracesResponse(BaseModel): + traces: List[TraceWithEvaluations] + count: int + total_count: int + + +class UploadURLResponse(BaseModel): + organization_id: str + project_id: str + url: str + + +class CreateTracesResponse(BaseModel): + trace_ids: List[str] + + +class TraceEvaluationsResponse(BaseModel): + trace_evaluations: List[TraceEvaluation] + count: int + total: int + + +class TraceEvaluationResultsResponse(BaseModel): + results: List[TraceEvaluationResult] + + +class CostEstimateResponse(BaseModel): + estimated_cost: float + input_tokens: int + output_tokens: int + model: str + trace_count: int + + +class JudgeOptimizationRunsResponse(BaseModel): + optimization_runs: List[JudgeOptimizationRun] + count: int + total: int + + +class CreateJudgeOptimizationRunResponse(BaseModel): + id: str + judge_id: str + budget: str + status: str + + +class EstimateJudgeOptimizationCostResponse(BaseModel): + estimated_cost: float + annotation_count: int + budget: str + + +class ApplyJudgeOptimizationResultResponse(BaseModel): + judge_id: str + new_version: int + message: str diff --git a/src/layerlens/models/evaluation.py b/src/layerlens/models/evaluation.py index fc5b2bd..692d4b6 100644 --- a/src/layerlens/models/evaluation.py +++ b/src/layerlens/models/evaluation.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from .api import ResultsResponse - from .._client import Atlas, AsyncAtlas + from .._client import Stratix, AsyncStratix class EvaluationStatus(str, Enum): @@ -33,9 +33,9 @@ class Evaluation(BaseModel): average_duration: int accuracy: float - _client: "Optional[Atlas | AsyncAtlas]" = None + _client: "Optional[Stratix | AsyncStratix]" = None - def attach_client(self, client: "Atlas | AsyncAtlas") -> "Evaluation": + def attach_client(self, client: "Stratix | AsyncStratix") -> "Evaluation": self._client = client return self @@ -61,11 +61,11 @@ def get_results( timeout: float | httpx.Timeout | None = None, ) -> Optional[ResultsResponse]: """Fetch results synchronously if a sync client is attached.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if isinstance(self._client, AsyncAtlas): + if isinstance(self._client, AsyncStratix): raise RuntimeError("Use `await get_results_async()` with an async client") return self._client.results.get(evaluation=self, page=page, page_size=page_size, timeout=timeout) @@ -76,11 +76,11 @@ def get_all_results( timeout: float | httpx.Timeout | None = None, ) -> List[Result]: """Fetch results synchronously if a sync client is attached.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if isinstance(self._client, AsyncAtlas): + if isinstance(self._client, AsyncStratix): raise RuntimeError("Use `await get_results_async()` with an async client") return self._client.results.get_all(evaluation=self, timeout=timeout) @@ -93,11 +93,11 @@ async def get_results_async( timeout: float | httpx.Timeout | None = None, ) -> Optional[ResultsResponse]: """Fetch results asynchronously if an async client is attached.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if not isinstance(self._client, AsyncAtlas): + if not isinstance(self._client, AsyncStratix): raise RuntimeError("Use `get_results()` with a sync client") return await self._client.results.get(evaluation=self, page=page, page_size=page_size, timeout=timeout) @@ -108,11 +108,11 @@ async def get_all_results_async( timeout: float | httpx.Timeout | None = None, ) -> List[Result]: """Fetch results asynchronously if an async client is attached.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if not isinstance(self._client, AsyncAtlas): + if not isinstance(self._client, AsyncStratix): raise RuntimeError("Use `get_results()` with a sync client") return await self._client.results.get_all(evaluation=self, timeout=timeout) @@ -121,11 +121,11 @@ def wait_for_completion( self, *, interval_seconds: int = 30, timeout_seconds: Optional[int] = None ) -> Optional["Evaluation"]: """Sync polling using a sync client.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if isinstance(self._client, AsyncAtlas): + if isinstance(self._client, AsyncStratix): raise RuntimeError("Use `wait_for_completion_async()` with an async client") evaluation = self._client.evaluations.wait_for_completion( @@ -143,11 +143,11 @@ async def wait_for_completion_async( self, *, interval_seconds: int = 30, timeout_seconds: Optional[int] = None ) -> Optional["Evaluation"]: """Async polling using an async client.""" - from .._client import AsyncAtlas + from .._client import AsyncStratix if self._client is None: raise ValueError("No client attached") - if not isinstance(self._client, AsyncAtlas): + if not isinstance(self._client, AsyncStratix): raise RuntimeError("Use `wait_for_completion()` with a sync client") evaluation = await self._client.evaluations.wait_for_completion( diff --git a/src/layerlens/models/judge.py b/src/layerlens/models/judge.py new file mode 100644 index 0000000..8b68439 --- /dev/null +++ b/src/layerlens/models/judge.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing import List, Optional + +from pydantic import BaseModel + + +class JudgeVersion(BaseModel): + version: int + name: str + evaluation_goal: str + model_id: Optional[str] = None + model_name: Optional[str] = None + model_company: Optional[str] = None + updated_at: str + updated_by: str + + +class Judge(BaseModel): + id: str + organization_id: str + project_id: str + name: str + evaluation_goal: str + model_id: Optional[str] = None + model_name: Optional[str] = None + model_company: Optional[str] = None + version: int + run_count: int = 0 + created_at: str + updated_at: str + versions: List[JudgeVersion] = [] diff --git a/src/layerlens/models/judge_optimization.py b/src/layerlens/models/judge_optimization.py new file mode 100644 index 0000000..7983464 --- /dev/null +++ b/src/layerlens/models/judge_optimization.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from pydantic import BaseModel + +from .trace_evaluation import JudgeSnapshot + + +class OptimizationRunStatus(str, Enum): + PENDING = "pending" + IN_PROGRESS = "in_progress" + SUCCESS = "success" + FAILURE = "failure" + + +class OptimizationBudget(str, Enum): + LIGHT = "light" + MEDIUM = "medium" + HEAVY = "heavy" + + +class JudgeOptimizationRun(BaseModel): + id: str + organization_id: str + project_id: str + judge_id: str + user_id: Optional[str] = None + status: OptimizationRunStatus + status_description: Optional[str] = None + judge_snapshot: Optional[JudgeSnapshot] = None + annotation_count: int = 0 + budget: OptimizationBudget = OptimizationBudget.MEDIUM + baseline_accuracy: Optional[float] = None + optimized_accuracy: Optional[float] = None + original_goal: Optional[str] = None + optimized_goal: Optional[str] = None + logs: Optional[str] = None + estimated_cost: float = 0.0 + actual_cost: float = 0.0 + created_at: str + started_at: Optional[str] = None + finished_at: Optional[str] = None + applied_at: Optional[str] = None + applied_version: Optional[int] = None diff --git a/src/layerlens/models/trace.py b/src/layerlens/models/trace.py new file mode 100644 index 0000000..50eeeb5 --- /dev/null +++ b/src/layerlens/models/trace.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + + +class Trace(BaseModel): + id: str + organization_id: str + project_id: str + created_at: str + filename: str + data: Dict[str, Any] = {} + input: Optional[str] = None + integration_id: Optional[str] = None + + +class TraceEvaluationSummary(BaseModel): + judge_id: str + judge_name: str + judge_version: int + created_at: str + passed: Optional[bool] = None + + +class TraceWithEvaluations(Trace): + evaluations_count: int = 0 + last_evaluations: List[TraceEvaluationSummary] = [] diff --git a/src/layerlens/models/trace_evaluation.py b/src/layerlens/models/trace_evaluation.py new file mode 100644 index 0000000..054044c --- /dev/null +++ b/src/layerlens/models/trace_evaluation.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from enum import Enum +from typing import List, Optional + +from pydantic import Field, BaseModel, ConfigDict + + +class TraceEvaluationStatus(str, Enum): + PENDING = "pending" + IN_PROGRESS = "in_progress" + SUCCESS = "success" + FAILURE = "failure" + + +class JudgeSnapshot(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + name: str + version: int + evaluation_goal: str = Field(..., alias="evaluationGoal") + model_id: Optional[str] = Field(None, alias="modelId") + model_name: Optional[str] = Field(None, alias="modelName") + model_company: Optional[str] = Field(None, alias="modelCompany") + + +class TraceEvaluation(BaseModel): + id: str + trace_id: str + judge_id: str + status: TraceEvaluationStatus + judge_snapshot: Optional[JudgeSnapshot] = None + created_at: Optional[str] = None + started_at: Optional[str] = None + finished_at: Optional[str] = None + + +class TraceEvaluationStep(BaseModel): + step: int + reasoning: str + + +class TraceEvaluationResult(BaseModel): + id: str + trace_evaluation_id: str + trace_id: str + judge_id: str + score: float + passed: bool + reasoning: str + steps: List[TraceEvaluationStep] = [] + model: str + turns: int + latency_ms: int + prompt_tokens: int + completion_tokens: int + total_cost: float + created_at: str diff --git a/src/layerlens/resources/judge_optimizations/__init__.py b/src/layerlens/resources/judge_optimizations/__init__.py new file mode 100644 index 0000000..4f676f2 --- /dev/null +++ b/src/layerlens/resources/judge_optimizations/__init__.py @@ -0,0 +1,3 @@ +from .judge_optimizations import JudgeOptimizations, AsyncJudgeOptimizations + +__all__ = ["JudgeOptimizations", "AsyncJudgeOptimizations"] diff --git a/src/layerlens/resources/judge_optimizations/judge_optimizations.py b/src/layerlens/resources/judge_optimizations/judge_optimizations.py new file mode 100644 index 0000000..bec3b8a --- /dev/null +++ b/src/layerlens/resources/judge_optimizations/judge_optimizations.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +import httpx + +from ...models import ( + JudgeOptimizationRun, + JudgeOptimizationRunsResponse, + CreateJudgeOptimizationRunResponse, + ApplyJudgeOptimizationResultResponse, + EstimateJudgeOptimizationCostResponse, +) +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._constants import DEFAULT_TIMEOUT + +DEFAULT_PAGE = 1 +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 500 + + +def _unwrap(resp: Any) -> Any: + """Unwrap {"status": ..., "data": ...} envelope if present.""" + if isinstance(resp, dict) and "data" in resp and "status" in resp: + return resp["data"] + return resp + + +class JudgeOptimizations(SyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/judge-optimizations" + + def estimate( + self, + *, + judge_id: str, + budget: str = "medium", + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[EstimateJudgeOptimizationCostResponse]: + body: Dict[str, Any] = { + "judge_id": judge_id, + "budget": budget, + } + + resp = self._post( + f"{self._base_url()}/estimate", + body=body, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return EstimateJudgeOptimizationCostResponse(**data) + except Exception: + return None + return None + + def create( + self, + *, + judge_id: str, + budget: str = "medium", + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CreateJudgeOptimizationRunResponse]: + body: Dict[str, Any] = { + "judge_id": judge_id, + "budget": budget, + } + + resp = self._post( + self._base_url(), + body=body, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return CreateJudgeOptimizationRunResponse(**data) + except Exception: + return None + return None + + def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgeOptimizationRun]: + resp = self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return JudgeOptimizationRun(**data) + except Exception: + return None + return None + + def get_many( + self, + *, + judge_id: Optional[str] = None, + page: Optional[int] = None, + page_size: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgeOptimizationRunsResponse]: + params: Dict[str, str] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["page_size"] = str(effective_page_size) + + if judge_id is not None: + params["judge_id"] = judge_id + + resp = self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + runs = [ + r if isinstance(r, JudgeOptimizationRun) else JudgeOptimizationRun(**r) + for r in data.get("optimization_runs", []) + ] + count: int = data.get("count", len(runs)) + total: int = data.get("total", count) + + try: + return JudgeOptimizationRunsResponse(optimization_runs=runs, count=count, total=total) + except Exception: + return None + + def apply( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[ApplyJudgeOptimizationResultResponse]: + resp = self._post( + f"{self._base_url()}/{id}/apply", + body={}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return ApplyJudgeOptimizationResultResponse(**data) + except Exception: + return None + return None + + +class AsyncJudgeOptimizations(AsyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/judge-optimizations" + + async def estimate( + self, + *, + judge_id: str, + budget: str = "medium", + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[EstimateJudgeOptimizationCostResponse]: + body: Dict[str, Any] = { + "judge_id": judge_id, + "budget": budget, + } + + resp = await self._post( + f"{self._base_url()}/estimate", + body=body, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return EstimateJudgeOptimizationCostResponse(**data) + except Exception: + return None + return None + + async def create( + self, + *, + judge_id: str, + budget: str = "medium", + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CreateJudgeOptimizationRunResponse]: + body: Dict[str, Any] = { + "judge_id": judge_id, + "budget": budget, + } + + resp = await self._post( + self._base_url(), + body=body, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return CreateJudgeOptimizationRunResponse(**data) + except Exception: + return None + return None + + async def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgeOptimizationRun]: + resp = await self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return JudgeOptimizationRun(**data) + except Exception: + return None + return None + + async def get_many( + self, + *, + judge_id: Optional[str] = None, + page: Optional[int] = None, + page_size: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgeOptimizationRunsResponse]: + params: Dict[str, str] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["page_size"] = str(effective_page_size) + + if judge_id is not None: + params["judge_id"] = judge_id + + resp = await self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + runs = [ + r if isinstance(r, JudgeOptimizationRun) else JudgeOptimizationRun(**r) + for r in data.get("optimization_runs", []) + ] + count: int = data.get("count", len(runs)) + total: int = data.get("total", count) + + try: + return JudgeOptimizationRunsResponse(optimization_runs=runs, count=count, total=total) + except Exception: + return None + + async def apply( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[ApplyJudgeOptimizationResultResponse]: + resp = await self._post( + f"{self._base_url()}/{id}/apply", + body={}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return ApplyJudgeOptimizationResultResponse(**data) + except Exception: + return None + return None diff --git a/src/layerlens/resources/judges/__init__.py b/src/layerlens/resources/judges/__init__.py new file mode 100644 index 0000000..e06ce06 --- /dev/null +++ b/src/layerlens/resources/judges/__init__.py @@ -0,0 +1,3 @@ +from .judges import Judges, AsyncJudges + +__all__ = ["Judges", "AsyncJudges"] diff --git a/src/layerlens/resources/judges/judges.py b/src/layerlens/resources/judges/judges.py new file mode 100644 index 0000000..3954204 --- /dev/null +++ b/src/layerlens/resources/judges/judges.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +import httpx + +from ...models import ( + Judge, + JudgesResponse, + DeleteJudgeResponse, + UpdateJudgeResponse, +) +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._constants import DEFAULT_TIMEOUT + +DEFAULT_PAGE = 1 +DEFAULT_PAGE_SIZE = 100 +MAX_PAGE_SIZE = 500 + + +class Judges(SyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/judges" + + def create( + self, + *, + name: str, + evaluation_goal: str, + model_id: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Judge]: + body: Dict[str, Any] = { + "name": name, + "evaluation_goal": evaluation_goal, + } + if model_id is not None: + body["model_id"] = model_id + + resp = self._post( + self._base_url(), + body=body, + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + judge_id = data.get("id") if isinstance(data, dict) else None + if judge_id: + return self.get(judge_id, timeout=timeout) + return None + + def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Judge]: + resp = self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return Judge(**data) if isinstance(data, dict) else None + except Exception: + return None + return None + + def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgesResponse]: + params: dict[str, str] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["pageSize"] = str(effective_page_size) + + resp = self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = resp.get("data", resp) if "status" in resp and "data" in resp else resp + if not isinstance(data, dict): + return None + + judges = [j if isinstance(j, Judge) else Judge(**j) for j in data.get("judges", [])] + count: int = data.get("count", len(judges)) + total_count: int = data.get("total_count", count) + + try: + return JudgesResponse(judges=judges, count=count, total_count=total_count) + except Exception: + return None + + def update( + self, + id: str, + *, + name: Optional[str] = None, + evaluation_goal: Optional[str] = None, + model_id: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[UpdateJudgeResponse]: + body: Dict[str, Any] = {} + if name is not None: + body["name"] = name + if evaluation_goal is not None: + body["evaluation_goal"] = evaluation_goal + if model_id is not None: + body["model_id"] = model_id + + resp = self._patch( + f"{self._base_url()}/{id}", + body=body, + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return UpdateJudgeResponse(**data) if isinstance(data, dict) else None + except Exception: + return None + return None + + def delete( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[DeleteJudgeResponse]: + resp = self._delete( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return DeleteJudgeResponse(**data) if isinstance(data, dict) else None + except Exception: + return None + return None + + +class AsyncJudges(AsyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/judges" + + async def create( + self, + *, + name: str, + evaluation_goal: str, + model_id: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Judge]: + body: Dict[str, Any] = { + "name": name, + "evaluation_goal": evaluation_goal, + } + if model_id is not None: + body["model_id"] = model_id + + resp = await self._post( + self._base_url(), + body=body, + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + judge_id = data.get("id") if isinstance(data, dict) else None + if judge_id: + return await self.get(judge_id, timeout=timeout) + return None + + async def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Judge]: + resp = await self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return Judge(**data) if isinstance(data, dict) else None + except Exception: + return None + return None + + async def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[JudgesResponse]: + params: dict[str, str] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["pageSize"] = str(effective_page_size) + + resp = await self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = resp.get("data", resp) if "status" in resp and "data" in resp else resp + if not isinstance(data, dict): + return None + + judges = [j if isinstance(j, Judge) else Judge(**j) for j in data.get("judges", [])] + count: int = data.get("count", len(judges)) + total_count: int = data.get("total_count", count) + + try: + return JudgesResponse(judges=judges, count=count, total_count=total_count) + except Exception: + return None + + async def update( + self, + id: str, + *, + name: Optional[str] = None, + evaluation_goal: Optional[str] = None, + model_id: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[UpdateJudgeResponse]: + body: Dict[str, Any] = {} + if name is not None: + body["name"] = name + if evaluation_goal is not None: + body["evaluation_goal"] = evaluation_goal + if model_id is not None: + body["model_id"] = model_id + + resp = await self._patch( + f"{self._base_url()}/{id}", + body=body, + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return UpdateJudgeResponse(**data) if isinstance(data, dict) else None + except Exception: + return None + return None + + async def delete( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[DeleteJudgeResponse]: + resp = await self._delete( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + if isinstance(resp, dict): + data = resp.get("data", resp) + try: + return DeleteJudgeResponse(**data) if isinstance(data, dict) else None + except Exception: + return None + return None diff --git a/src/layerlens/resources/trace_evaluations/__init__.py b/src/layerlens/resources/trace_evaluations/__init__.py new file mode 100644 index 0000000..53c6b5d --- /dev/null +++ b/src/layerlens/resources/trace_evaluations/__init__.py @@ -0,0 +1,3 @@ +from .trace_evaluations import TraceEvaluations, AsyncTraceEvaluations + +__all__ = ["TraceEvaluations", "AsyncTraceEvaluations"] diff --git a/src/layerlens/resources/trace_evaluations/trace_evaluations.py b/src/layerlens/resources/trace_evaluations/trace_evaluations.py new file mode 100644 index 0000000..484f657 --- /dev/null +++ b/src/layerlens/resources/trace_evaluations/trace_evaluations.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import httpx + +from ...models import ( + TraceEvaluation, + CostEstimateResponse, + TraceEvaluationResult, + TraceEvaluationsResponse, + TraceEvaluationResultsResponse, +) +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._constants import DEFAULT_TIMEOUT + +DEFAULT_PAGE = 1 +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 + + +def _unwrap(resp: Any) -> Any: + """Unwrap {"status": ..., "data": ...} envelope if present.""" + if isinstance(resp, dict) and "data" in resp and "status" in resp: + return resp["data"] + return resp + + +class TraceEvaluations(SyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/trace-evaluations" + + def create( + self, + *, + trace_id: str, + judge_id: str, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluation]: + resp = self._post( + self._base_url(), + body={"trace_id": trace_id, "judge_id": judge_id}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return TraceEvaluation(**data) + except Exception: + return None + return None + + def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluation]: + resp = self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return TraceEvaluation(**data) + except Exception: + return None + return None + + def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + judge_id: Optional[str] = None, + trace_id: Optional[str] = None, + outcome: Optional[str] = None, + time_range: Optional[str] = None, + search: Optional[str] = None, + sort_by: Optional[str] = None, + sort_order: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluationsResponse]: + params: Dict[str, Any] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["page_size"] = str(effective_page_size) + + if judge_id is not None: + params["judgeId"] = judge_id + if trace_id is not None: + params["traceId"] = trace_id + if outcome is not None: + params["outcome"] = outcome + if time_range is not None: + params["timeRange"] = time_range + if search is not None: + params["search"] = search + if sort_by is not None: + params["sortBy"] = sort_by + if sort_order is not None: + params["sortOrder"] = sort_order + + resp = self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + evaluations = [ + te if isinstance(te, TraceEvaluation) else TraceEvaluation(**te) for te in data.get("trace_evaluations", []) + ] + count: int = data.get("count", len(evaluations)) + total: int = data.get("total", count) + + try: + return TraceEvaluationsResponse(trace_evaluations=evaluations, count=count, total=total) + except Exception: + return None + + def get_results( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluationResultsResponse]: + resp = self._get( + f"{self._base_url()}/{id}/results", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if not data or not isinstance(data, dict): + return None + + results = [ + r if isinstance(r, TraceEvaluationResult) else TraceEvaluationResult(**r) for r in data.get("results", []) + ] + + try: + return TraceEvaluationResultsResponse(results=results) + except Exception: + return None + + def estimate_cost( + self, + *, + trace_ids: List[str], + judge_id: str, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CostEstimateResponse]: + resp = self._post( + f"{self._base_url()}/estimate", + body={"trace_ids": trace_ids, "judge_id": judge_id}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return CostEstimateResponse(**data) + except Exception: + return None + return None + + +class AsyncTraceEvaluations(AsyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/trace-evaluations" + + async def create( + self, + *, + trace_id: str, + judge_id: str, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluation]: + resp = await self._post( + self._base_url(), + body={"trace_id": trace_id, "judge_id": judge_id}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return TraceEvaluation(**data) + except Exception: + return None + return None + + async def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluation]: + resp = await self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return TraceEvaluation(**data) + except Exception: + return None + return None + + async def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + judge_id: Optional[str] = None, + trace_id: Optional[str] = None, + outcome: Optional[str] = None, + time_range: Optional[str] = None, + search: Optional[str] = None, + sort_by: Optional[str] = None, + sort_order: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluationsResponse]: + params: Dict[str, Any] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["page_size"] = str(effective_page_size) + + if judge_id is not None: + params["judgeId"] = judge_id + if trace_id is not None: + params["traceId"] = trace_id + if outcome is not None: + params["outcome"] = outcome + if time_range is not None: + params["timeRange"] = time_range + if search is not None: + params["search"] = search + if sort_by is not None: + params["sortBy"] = sort_by + if sort_order is not None: + params["sortOrder"] = sort_order + + resp = await self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + evaluations = [ + te if isinstance(te, TraceEvaluation) else TraceEvaluation(**te) for te in data.get("trace_evaluations", []) + ] + count: int = data.get("count", len(evaluations)) + total: int = data.get("total", count) + + try: + return TraceEvaluationsResponse(trace_evaluations=evaluations, count=count, total=total) + except Exception: + return None + + async def get_results( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TraceEvaluationResultsResponse]: + resp = await self._get( + f"{self._base_url()}/{id}/results", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if not data or not isinstance(data, dict): + return None + + results = [ + r if isinstance(r, TraceEvaluationResult) else TraceEvaluationResult(**r) for r in data.get("results", []) + ] + + try: + return TraceEvaluationResultsResponse(results=results) + except Exception: + return None + + async def estimate_cost( + self, + *, + trace_ids: List[str], + judge_id: str, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CostEstimateResponse]: + resp = await self._post( + f"{self._base_url()}/estimate", + body={"trace_ids": trace_ids, "judge_id": judge_id}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return CostEstimateResponse(**data) + except Exception: + return None + return None diff --git a/src/layerlens/resources/traces/__init__.py b/src/layerlens/resources/traces/__init__.py new file mode 100644 index 0000000..6639aed --- /dev/null +++ b/src/layerlens/resources/traces/__init__.py @@ -0,0 +1,3 @@ +from .traces import Traces, AsyncTraces + +__all__ = ["Traces", "AsyncTraces"] diff --git a/src/layerlens/resources/traces/traces.py b/src/layerlens/resources/traces/traces.py new file mode 100644 index 0000000..5e5012c --- /dev/null +++ b/src/layerlens/resources/traces/traces.py @@ -0,0 +1,385 @@ +from __future__ import annotations + +import os +import mimetypes +from typing import Any, Dict, List, Optional + +import httpx + +from ...models import ( + Trace, + TracesResponse, + CreateTracesResponse, + TraceWithEvaluations, +) +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._constants import DEFAULT_TIMEOUT + +DEFAULT_PAGE = 1 +DEFAULT_PAGE_SIZE = 100 +MAX_PAGE_SIZE = 500 + +MAX_UPLOAD_SIZE = 50 * 1024 * 1024 # 50 MB + + +def _unwrap(resp: Any) -> Any: + """Unwrap {"status": ..., "data": ...} envelope if present.""" + if isinstance(resp, dict) and "data" in resp and "status" in resp: + return resp["data"] + return resp + + +class Traces(SyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/traces" + + def upload( + self, + file_path: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CreateTracesResponse]: + """Upload a JSON or JSONL trace file. + + Three-step flow: + 1. Request a presigned upload URL from the API + 2. Upload the file to S3 via the presigned URL + 3. Create trace records from the uploaded file + """ + file_path = os.path.abspath(file_path) + filename = os.path.basename(file_path) + file_size = os.path.getsize(file_path) + + if file_size > MAX_UPLOAD_SIZE: + raise ValueError(f"File size {file_size} exceeds maximum of {MAX_UPLOAD_SIZE} bytes (50 MB)") + + content_type = _get_content_type(filename) + + # Step 1: Get presigned upload URL + raw_resp = self._post( + f"{self._base_url()}/upload", + body={"filename": filename, "type": content_type, "size": file_size}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(raw_resp) + if not isinstance(data, dict) or "url" not in data: + return None + upload_url: str = data["url"] + + # Step 2: Upload file to S3 + with open(file_path, "rb") as f: + put_resp = httpx.put( + upload_url, + content=f.read(), + headers={"Content-Type": content_type}, + timeout=timeout if isinstance(timeout, httpx.Timeout) else httpx.Timeout(timeout), + ) + put_resp.raise_for_status() + + # Step 3: Create trace records + raw_create = self._post( + self._base_url(), + body={"filename": filename}, + timeout=timeout, + cast_to=dict, + ) + create_data = _unwrap(raw_create) + if isinstance(create_data, list): + return CreateTracesResponse(trace_ids=create_data) + if isinstance(create_data, dict): + try: + return CreateTracesResponse(**create_data) + except Exception: + return None + return None + + def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Trace]: + resp = self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return Trace(**data) + except Exception: + return None + return None + + def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + source: Optional[str] = None, + judge_id: Optional[str] = None, + status: Optional[str] = None, + time_range: Optional[str] = None, + search: Optional[str] = None, + sort_by: Optional[str] = None, + sort_order: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TracesResponse]: + params: Dict[str, Any] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["pageSize"] = str(effective_page_size) + + if source is not None: + params["source"] = source + if judge_id is not None: + params["judgeId"] = judge_id + if status is not None: + params["status"] = status + if time_range is not None: + params["timeRange"] = time_range + if search is not None: + params["search"] = search + if sort_by is not None: + params["sortBy"] = sort_by + if sort_order is not None: + params["sortOrder"] = sort_order + + resp = self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + traces = [ + t if isinstance(t, TraceWithEvaluations) else TraceWithEvaluations(**t) for t in data.get("traces", []) + ] + count: int = data.get("count", len(traces)) + total_count: int = data.get("total_count", count) + + try: + return TracesResponse(traces=traces, count=count, total_count=total_count) + except Exception: + return None + + def delete( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + resp = self._delete( + f"{self._base_url()}/{id}", + timeout=timeout, + ) + return resp is not None + + def get_sources( + self, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> List[str]: + resp = self._get( + f"{self._base_url()}/sources", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + return list(data.get("sources", [])) + return [] + + +class AsyncTraces(AsyncAPIResource): + def _base_url(self) -> str: + return f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/traces" + + async def upload( + self, + file_path: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[CreateTracesResponse]: + """Upload a JSON or JSONL trace file. + + Three-step flow: + 1. Request a presigned upload URL from the API + 2. Upload the file to S3 via the presigned URL + 3. Create trace records from the uploaded file + """ + file_path = os.path.abspath(file_path) + filename = os.path.basename(file_path) + file_size = os.path.getsize(file_path) + + if file_size > MAX_UPLOAD_SIZE: + raise ValueError(f"File size {file_size} exceeds maximum of {MAX_UPLOAD_SIZE} bytes (50 MB)") + + content_type = _get_content_type(filename) + + # Step 1: Get presigned upload URL + raw_resp = await self._post( + f"{self._base_url()}/upload", + body={"filename": filename, "type": content_type, "size": file_size}, + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(raw_resp) + if not isinstance(data, dict) or "url" not in data: + return None + upload_url: str = data["url"] + + # Step 2: Upload file to S3 + async with httpx.AsyncClient() as upload_client: + with open(file_path, "rb") as f: + put_resp = await upload_client.put( + upload_url, + content=f.read(), + headers={"Content-Type": content_type}, + timeout=timeout if isinstance(timeout, httpx.Timeout) else httpx.Timeout(timeout), + ) + put_resp.raise_for_status() + + # Step 3: Create trace records + raw_create = await self._post( + self._base_url(), + body={"filename": filename}, + timeout=timeout, + cast_to=dict, + ) + create_data = _unwrap(raw_create) + if isinstance(create_data, list): + return CreateTracesResponse(trace_ids=create_data) + if isinstance(create_data, dict): + try: + return CreateTracesResponse(**create_data) + except Exception: + return None + return None + + async def get( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[Trace]: + resp = await self._get( + f"{self._base_url()}/{id}", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + try: + return Trace(**data) + except Exception: + return None + return None + + async def get_many( + self, + *, + page: Optional[int] = None, + page_size: Optional[int] = None, + source: Optional[str] = None, + judge_id: Optional[str] = None, + status: Optional[str] = None, + time_range: Optional[str] = None, + search: Optional[str] = None, + sort_by: Optional[str] = None, + sort_order: Optional[str] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> Optional[TracesResponse]: + params: Dict[str, Any] = {} + + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE + effective_page = page if page is not None else DEFAULT_PAGE + + params["page"] = str(effective_page) + params["pageSize"] = str(effective_page_size) + + if source is not None: + params["source"] = source + if judge_id is not None: + params["judgeId"] = judge_id + if status is not None: + params["status"] = status + if time_range is not None: + params["timeRange"] = time_range + if search is not None: + params["search"] = search + if sort_by is not None: + params["sortBy"] = sort_by + if sort_order is not None: + params["sortOrder"] = sort_order + + resp = await self._get( + self._base_url(), + params=params, + timeout=timeout, + cast_to=dict, + ) + if not resp or not isinstance(resp, dict): + return None + + data = _unwrap(resp) + if not isinstance(data, dict): + return None + + traces = [ + t if isinstance(t, TraceWithEvaluations) else TraceWithEvaluations(**t) for t in data.get("traces", []) + ] + count: int = data.get("count", len(traces)) + total_count: int = data.get("total_count", count) + + try: + return TracesResponse(traces=traces, count=count, total_count=total_count) + except Exception: + return None + + async def delete( + self, + id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + resp = await self._delete( + f"{self._base_url()}/{id}", + timeout=timeout, + ) + return resp is not None + + async def get_sources( + self, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> List[str]: + resp = await self._get( + f"{self._base_url()}/sources", + timeout=timeout, + cast_to=dict, + ) + data = _unwrap(resp) + if isinstance(data, dict): + return list(data.get("sources", [])) + return [] + + +def _get_content_type(filename: str) -> str: + if filename.endswith(".jsonl"): + return "application/jsonl" + if filename.endswith(".json"): + return "application/json" + mime_type, _ = mimetypes.guess_type(filename) + return mime_type or "application/json" diff --git a/tests/conftest.py b/tests/conftest.py index 89eae24..16b8ed4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ def env_vars(): """Clean environment variables for testing.""" env_keys = [ - "LAYERLENS_ATLAS_API_KEY", + "LAYERLENS_STRATIX_API_KEY", ] original_values = {key: os.environ.get(key) for key in env_keys} @@ -33,7 +33,7 @@ def mock_env_vars(): with mock.patch.dict( os.environ, { - "LAYERLENS_ATLAS_API_KEY": "test-api-key", + "LAYERLENS_STRATIX_API_KEY": "test-api-key", }, ): yield diff --git a/tests/resources/test_benchmarks.py b/tests/resources/test_benchmarks.py index d9b1c5b..2155105 100644 --- a/tests/resources/test_benchmarks.py +++ b/tests/resources/test_benchmarks.py @@ -18,7 +18,7 @@ class TestBenchmarks: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" @@ -304,7 +304,7 @@ class TestBenchmarksErrorHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" @@ -372,7 +372,7 @@ class TestBenchmarksTyping: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" diff --git a/tests/resources/test_evaluations.py b/tests/resources/test_evaluations.py index 4cffcb9..02480a3 100644 --- a/tests/resources/test_evaluations.py +++ b/tests/resources/test_evaluations.py @@ -18,7 +18,7 @@ class TestEvaluations: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" @@ -325,7 +325,7 @@ class TestEvaluationsErrorHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" diff --git a/tests/resources/test_judge_optimizations.py b/tests/resources/test_judge_optimizations.py new file mode 100644 index 0000000..7d426fe --- /dev/null +++ b/tests/resources/test_judge_optimizations.py @@ -0,0 +1,481 @@ +from unittest.mock import Mock + +import pytest + +from layerlens.models import ( + JudgeOptimizationRun, + JudgeOptimizationRunsResponse, + CreateJudgeOptimizationRunResponse, + ApplyJudgeOptimizationResultResponse, + EstimateJudgeOptimizationCostResponse, +) +from layerlens._constants import DEFAULT_TIMEOUT +from layerlens.resources.judge_optimizations.judge_optimizations import JudgeOptimizations + + +class TestJudgeOptimizations: + """Test JudgeOptimizations resource API methods.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def resource(self, mock_client): + """JudgeOptimizations resource instance.""" + return JudgeOptimizations(mock_client) + + @pytest.fixture + def sample_run_data(self): + """Sample optimization run data.""" + return { + "id": "run-123", + "organization_id": "org-123", + "project_id": "proj-456", + "judge_id": "judge-789", + "user_id": "user-001", + "status": "pending", + "status_description": None, + "judge_snapshot": None, + "annotation_count": 50, + "budget": "medium", + "baseline_accuracy": None, + "optimized_accuracy": None, + "original_goal": None, + "optimized_goal": None, + "logs": None, + "estimated_cost": 7.5, + "actual_cost": 0.0, + "created_at": "2024-06-01T00:00:00Z", + "started_at": None, + "finished_at": None, + "applied_at": None, + "applied_version": None, + } + + @pytest.fixture + def sample_completed_run_data(self): + """Sample completed optimization run data.""" + return { + "id": "run-456", + "organization_id": "org-123", + "project_id": "proj-456", + "judge_id": "judge-789", + "user_id": "user-001", + "status": "success", + "status_description": None, + "judge_snapshot": { + "name": "Code Quality Judge", + "version": 1, + "evaluationGoal": "Evaluate code quality", + "modelId": None, + "modelName": None, + "modelCompany": None, + }, + "annotation_count": 50, + "budget": "medium", + "baseline_accuracy": 0.75, + "optimized_accuracy": 0.82, + "original_goal": "Evaluate code quality", + "optimized_goal": "Evaluate code quality with emphasis on correctness, style, and maintainability", + "logs": "[INFO] Starting optimization...\n[METRIC] Baseline: 0.75\n[METRIC] Optimized: 0.82", + "estimated_cost": 7.5, + "actual_cost": 5.25, + "created_at": "2024-06-01T00:00:00Z", + "started_at": "2024-06-01T00:01:00Z", + "finished_at": "2024-06-01T00:10:00Z", + "applied_at": None, + "applied_version": None, + } + + # --- estimate --- + + def test_estimate_success(self, resource): + """estimate returns cost estimate on success.""" + resource._post.return_value = {"estimated_cost": 7.5, "annotation_count": 50, "budget": "medium"} + + result = resource.estimate(judge_id="judge-789", budget="medium") + + assert isinstance(result, EstimateJudgeOptimizationCostResponse) + assert result.estimated_cost == 7.5 + assert result.annotation_count == 50 + assert result.budget == "medium" + + def test_estimate_request_parameters(self, resource): + """estimate makes correct API request.""" + resource._post.return_value = {"estimated_cost": 7.5, "annotation_count": 50, "budget": "medium"} + + resource.estimate(judge_id="judge-789", budget="heavy") + + resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judge-optimizations/estimate", + body={"judge_id": "judge-789", "budget": "heavy"}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_estimate_default_budget(self, resource): + """estimate uses medium budget by default.""" + resource._post.return_value = {"estimated_cost": 7.5, "annotation_count": 50, "budget": "medium"} + + resource.estimate(judge_id="judge-789") + + call_args = resource._post.call_args + assert call_args.kwargs["body"]["budget"] == "medium" + + def test_estimate_none_response(self, resource): + """estimate returns None when response is invalid.""" + resource._post.return_value = None + + result = resource.estimate(judge_id="judge-789") + + assert result is None + + # --- create --- + + def test_create_success(self, resource): + """create returns CreateJudgeOptimizationRunResponse on success.""" + resource._post.return_value = { + "id": "run-123", + "judge_id": "judge-789", + "budget": "medium", + "status": "pending", + } + + result = resource.create(judge_id="judge-789", budget="medium") + + assert isinstance(result, CreateJudgeOptimizationRunResponse) + assert result.id == "run-123" + assert result.judge_id == "judge-789" + assert result.status == "pending" + + def test_create_request_parameters(self, resource): + """create makes correct API request.""" + resource._post.return_value = {"id": "run-123", "judge_id": "judge-789", "budget": "light", "status": "pending"} + + resource.create(judge_id="judge-789", budget="light") + + resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judge-optimizations", + body={"judge_id": "judge-789", "budget": "light"}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_create_none_response(self, resource): + """create returns None when response is invalid.""" + resource._post.return_value = None + + result = resource.create(judge_id="judge-789") + + assert result is None + + # --- get --- + + def test_get_success(self, resource, sample_run_data): + """get returns JudgeOptimizationRun on success.""" + resource._get.return_value = sample_run_data + + result = resource.get("run-123") + + assert isinstance(result, JudgeOptimizationRun) + assert result.id == "run-123" + assert result.judge_id == "judge-789" + assert result.status.value == "pending" + + def test_get_completed_run(self, resource, sample_completed_run_data): + """get returns completed run with results.""" + resource._get.return_value = sample_completed_run_data + + result = resource.get("run-456") + + assert isinstance(result, JudgeOptimizationRun) + assert result.status.value == "success" + assert result.baseline_accuracy == 0.75 + assert result.optimized_accuracy == 0.82 + assert result.optimized_goal is not None + + def test_get_request_parameters(self, resource, sample_run_data): + """get makes correct API request.""" + resource._get.return_value = sample_run_data + + resource.get("run-123") + + resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judge-optimizations/run-123", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_none_response(self, resource): + """get returns None when response is invalid.""" + resource._get.return_value = None + + result = resource.get("run-123") + + assert result is None + + # --- get_many --- + + def test_get_many_success(self, resource, sample_run_data): + """get_many returns JudgeOptimizationRunsResponse on success.""" + resource._get.return_value = { + "optimization_runs": [sample_run_data], + "count": 1, + "total": 1, + } + + result = resource.get_many() + + assert isinstance(result, JudgeOptimizationRunsResponse) + assert len(result.optimization_runs) == 1 + assert result.optimization_runs[0].id == "run-123" + assert result.count == 1 + assert result.total == 1 + + def test_get_many_with_judge_id(self, resource, sample_run_data): + """get_many filters by judge_id.""" + resource._get.return_value = { + "optimization_runs": [sample_run_data], + "count": 1, + "total": 1, + } + + resource.get_many(judge_id="judge-789", page=1, page_size=10) + + resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judge-optimizations", + params={ + "page": "1", + "page_size": "10", + "judge_id": "judge-789", + }, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_many_default_pagination(self, resource, sample_run_data): + """get_many uses default pagination when not specified.""" + resource._get.return_value = { + "optimization_runs": [sample_run_data], + "count": 1, + "total": 1, + } + + resource.get_many() + + call_args = resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "1" + assert params["page_size"] == "20" + + def test_get_many_none_response(self, resource): + """get_many returns None when response is invalid.""" + resource._get.return_value = None + + result = resource.get_many() + + assert result is None + + def test_get_many_empty_list(self, resource): + """get_many returns empty list when no runs exist.""" + resource._get.return_value = { + "optimization_runs": [], + "count": 0, + "total": 0, + } + + result = resource.get_many() + + assert isinstance(result, JudgeOptimizationRunsResponse) + assert len(result.optimization_runs) == 0 + assert result.total == 0 + + # --- apply --- + + def test_apply_success(self, resource): + """apply returns ApplyJudgeOptimizationResultResponse on success.""" + resource._post.return_value = { + "judge_id": "judge-789", + "new_version": 3, + "message": "Optimization result applied successfully", + } + + result = resource.apply("run-456") + + assert isinstance(result, ApplyJudgeOptimizationResultResponse) + assert result.judge_id == "judge-789" + assert result.new_version == 3 + + def test_apply_request_parameters(self, resource): + """apply makes correct API request.""" + resource._post.return_value = { + "judge_id": "judge-789", + "new_version": 3, + "message": "Optimization result applied successfully", + } + + resource.apply("run-456") + + resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judge-optimizations/run-456/apply", + body={}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_apply_none_response(self, resource): + """apply returns None when response is invalid.""" + resource._post.return_value = None + + result = resource.apply("run-456") + + assert result is None + + +class TestJudgeOptimizationsErrorHandling: + """Test error handling in JudgeOptimizations resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def resource(self, mock_client): + """JudgeOptimizations resource instance.""" + return JudgeOptimizations(mock_client) + + def test_estimate_handles_bad_request(self, resource): + """estimate propagates bad request errors.""" + from layerlens._exceptions import BadRequestError + + mock_response = Mock() + mock_response.status_code = 400 + mock_response.headers = {} + + resource._post.side_effect = BadRequestError( + "Minimum 10 annotations required", response=mock_response, body=None + ) + + with pytest.raises(BadRequestError): + resource.estimate(judge_id="judge-789") + + def test_create_handles_bad_request(self, resource): + """create propagates bad request errors.""" + from layerlens._exceptions import BadRequestError + + mock_response = Mock() + mock_response.status_code = 400 + mock_response.headers = {} + + resource._post.side_effect = BadRequestError("Invalid budget value", response=mock_response, body=None) + + with pytest.raises(BadRequestError): + resource.create(judge_id="judge-789", budget="invalid") + + def test_get_handles_not_found(self, resource): + """get propagates not found errors.""" + from layerlens._exceptions import NotFoundError + + mock_response = Mock() + mock_response.status_code = 404 + mock_response.headers = {} + + resource._get.side_effect = NotFoundError("Optimization run not found", response=mock_response, body=None) + + with pytest.raises(NotFoundError): + resource.get("nonexistent-run") + + def test_apply_handles_conflict(self, resource): + """apply propagates conflict errors (already applied).""" + from layerlens._exceptions import ConflictError + + mock_response = Mock() + mock_response.status_code = 409 + mock_response.headers = {} + + resource._post.side_effect = ConflictError("Optimization already applied", response=mock_response, body=None) + + with pytest.raises(ConflictError): + resource.apply("run-456") + + def test_apply_handles_auth_error(self, resource): + """apply propagates authentication errors.""" + from layerlens._exceptions import AuthenticationError + + mock_response = Mock() + mock_response.status_code = 401 + mock_response.headers = {} + + resource._post.side_effect = AuthenticationError("Unauthorized", response=mock_response, body=None) + + with pytest.raises(AuthenticationError): + resource.apply("run-456") + + +class TestJudgeOptimizationsURLConstruction: + """Test URL construction in JudgeOptimizations resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "custom-org" + client.project_id = "custom-proj" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def resource(self, mock_client): + """JudgeOptimizations resource instance.""" + return JudgeOptimizations(mock_client) + + def test_base_url_construction(self, resource): + """Base URL uses correct organization and project IDs.""" + assert resource._base_url() == "/organizations/custom-org/projects/custom-proj/judge-optimizations" + + def test_get_url_includes_run_id(self, resource): + """Get URL includes run ID.""" + resource._get.return_value = None + + resource.get("run-abc") + + call_args = resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judge-optimizations/run-abc" + + def test_apply_url_includes_run_id(self, resource): + """Apply URL includes run ID and /apply suffix.""" + resource._post.return_value = None + + resource.apply("run-xyz") + + call_args = resource._post.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judge-optimizations/run-xyz/apply" + + def test_estimate_url(self, resource): + """Estimate URL includes /estimate suffix.""" + resource._post.return_value = None + + resource.estimate(judge_id="judge-123") + + call_args = resource._post.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judge-optimizations/estimate" diff --git a/tests/resources/test_judges.py b/tests/resources/test_judges.py new file mode 100644 index 0000000..91bb217 --- /dev/null +++ b/tests/resources/test_judges.py @@ -0,0 +1,448 @@ +from unittest.mock import Mock + +import pytest + +from layerlens.models import Judge, JudgesResponse, DeleteJudgeResponse, UpdateJudgeResponse +from layerlens._constants import DEFAULT_TIMEOUT +from layerlens.resources.judges.judges import Judges + + +class TestJudges: + """Test Judges resource API methods.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def judges_resource(self, mock_client): + """Judges resource instance.""" + return Judges(mock_client) + + @pytest.fixture + def sample_judge_data(self): + """Sample judge data for testing.""" + return { + "id": "judge-123", + "organization_id": "org-123", + "project_id": "proj-456", + "name": "Code Quality Judge", + "evaluation_goal": "Evaluate the quality of code output including correctness and style", + "model_id": "model-789", + "model_name": "GPT-4", + "model_company": "OpenAI", + "version": 1, + "run_count": 5, + "created_at": "2024-01-01T00:00:00Z", + "updated_at": "2024-01-02T00:00:00Z", + "versions": [], + } + + def test_judges_initialization(self, mock_client): + """Judges resource initializes correctly.""" + judges = Judges(mock_client) + + assert judges._client is mock_client + assert judges._get is mock_client.get_cast + assert judges._post is mock_client.post_cast + assert judges._patch is mock_client.patch_cast + assert judges._delete is mock_client.delete_cast + + def test_create_judge_success(self, judges_resource, sample_judge_data): + """create method returns Judge on success.""" + judges_resource._post.return_value = {"id": "judge-123"} + judges_resource._get.return_value = sample_judge_data + + result = judges_resource.create( + name="Code Quality Judge", + evaluation_goal="Evaluate the quality of code output including correctness and style", + model_id="model-789", + ) + + assert isinstance(result, Judge) + assert result.id == "judge-123" + assert result.name == "Code Quality Judge" + + def test_create_judge_request_parameters(self, judges_resource, sample_judge_data): + """create method makes correct API request.""" + judges_resource._post.return_value = {"id": "judge-123"} + judges_resource._get.return_value = sample_judge_data + + judges_resource.create( + name="Test Judge", + evaluation_goal="Test evaluation goal for testing", + model_id="model-789", + ) + + judges_resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judges", + body={ + "name": "Test Judge", + "evaluation_goal": "Test evaluation goal for testing", + "model_id": "model-789", + }, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_create_judge_without_model_id(self, judges_resource, sample_judge_data): + """create method works without model_id.""" + judges_resource._post.return_value = {"id": "judge-123"} + judges_resource._get.return_value = sample_judge_data + + judges_resource.create( + name="Test Judge", + evaluation_goal="Test evaluation goal for testing", + ) + + call_args = judges_resource._post.call_args + assert "model_id" not in call_args.kwargs["body"] + + def test_create_judge_none_response(self, judges_resource): + """create method returns None when response is not a dict.""" + judges_resource._post.return_value = None + + result = judges_resource.create( + name="Test Judge", + evaluation_goal="Test evaluation goal for testing", + ) + + assert result is None + + def test_get_judge_success(self, judges_resource, sample_judge_data): + """get method returns Judge on success.""" + judges_resource._get.return_value = sample_judge_data + + result = judges_resource.get("judge-123") + + assert isinstance(result, Judge) + assert result.id == "judge-123" + assert result.name == "Code Quality Judge" + + def test_get_judge_request_parameters(self, judges_resource, sample_judge_data): + """get method makes correct API request.""" + judges_resource._get.return_value = sample_judge_data + + judges_resource.get("judge-123") + + judges_resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judges/judge-123", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_judge_none_response(self, judges_resource): + """get method returns None when response is invalid.""" + judges_resource._get.return_value = None + + result = judges_resource.get("judge-123") + + assert result is None + + def test_get_many_judges_success(self, judges_resource, sample_judge_data): + """get_many returns JudgesResponse on success.""" + judges_resource._get.return_value = { + "judges": [sample_judge_data], + "count": 1, + "total_count": 1, + } + + result = judges_resource.get_many() + + assert isinstance(result, JudgesResponse) + assert len(result.judges) == 1 + assert result.judges[0].id == "judge-123" + assert result.count == 1 + assert result.total_count == 1 + + def test_get_many_judges_request_parameters(self, judges_resource, sample_judge_data): + """get_many makes correct API request with pagination.""" + judges_resource._get.return_value = { + "judges": [sample_judge_data], + "count": 1, + "total_count": 10, + } + + judges_resource.get_many(page=2, page_size=50) + + judges_resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judges", + params={ + "page": "2", + "pageSize": "50", + }, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_many_judges_default_pagination(self, judges_resource, sample_judge_data): + """get_many uses default pagination when not specified.""" + judges_resource._get.return_value = { + "judges": [sample_judge_data], + "count": 1, + "total_count": 1, + } + + judges_resource.get_many() + + call_args = judges_resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "1" + assert params["pageSize"] == "100" + + def test_get_many_judges_none_response(self, judges_resource): + """get_many returns None when response is invalid.""" + judges_resource._get.return_value = None + + result = judges_resource.get_many() + + assert result is None + + def test_get_many_judges_empty_list(self, judges_resource): + """get_many returns empty list when no judges exist.""" + judges_resource._get.return_value = { + "judges": [], + "count": 0, + "total_count": 0, + } + + result = judges_resource.get_many() + + assert isinstance(result, JudgesResponse) + assert len(result.judges) == 0 + assert result.total_count == 0 + + def test_update_judge_success(self, judges_resource): + """update method returns UpdateJudgeResponse on success.""" + judges_resource._patch.return_value = { + "organization_id": "org-123", + "project_id": "proj-456", + "id": "judge-123", + } + + result = judges_resource.update( + "judge-123", + name="Updated Judge", + evaluation_goal="Updated evaluation goal for the judge", + ) + + assert isinstance(result, UpdateJudgeResponse) + assert result.id == "judge-123" + + def test_update_judge_request_parameters(self, judges_resource): + """update method makes correct API request.""" + judges_resource._patch.return_value = { + "organization_id": "org-123", + "project_id": "proj-456", + "id": "judge-123", + } + + judges_resource.update( + "judge-123", + name="Updated Name", + evaluation_goal="Updated goal text for testing", + model_id="new-model", + ) + + judges_resource._patch.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judges/judge-123", + body={ + "name": "Updated Name", + "evaluation_goal": "Updated goal text for testing", + "model_id": "new-model", + }, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_update_judge_partial(self, judges_resource): + """update method only sends provided fields.""" + judges_resource._patch.return_value = { + "organization_id": "org-123", + "project_id": "proj-456", + "id": "judge-123", + } + + judges_resource.update("judge-123", name="New Name Only") + + call_args = judges_resource._patch.call_args + body = call_args.kwargs["body"] + assert body == {"name": "New Name Only"} + + def test_update_judge_none_response(self, judges_resource): + """update method returns None when response is invalid.""" + judges_resource._patch.return_value = None + + result = judges_resource.update("judge-123", name="Updated") + + assert result is None + + def test_delete_judge_success(self, judges_resource): + """delete method returns DeleteJudgeResponse on success.""" + judges_resource._delete.return_value = { + "organization_id": "org-123", + "project_id": "proj-456", + "id": "judge-123", + } + + result = judges_resource.delete("judge-123") + + assert isinstance(result, DeleteJudgeResponse) + assert result.id == "judge-123" + + def test_delete_judge_request_parameters(self, judges_resource): + """delete method makes correct API request.""" + judges_resource._delete.return_value = { + "organization_id": "org-123", + "project_id": "proj-456", + "id": "judge-123", + } + + judges_resource.delete("judge-123") + + judges_resource._delete.assert_called_once_with( + "/organizations/org-123/projects/proj-456/judges/judge-123", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_delete_judge_none_response(self, judges_resource): + """delete method returns None when response is invalid.""" + judges_resource._delete.return_value = None + + result = judges_resource.delete("judge-123") + + assert result is None + + +class TestJudgesErrorHandling: + """Test error handling in Judges resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def judges_resource(self, mock_client): + """Judges resource instance.""" + return Judges(mock_client) + + def test_get_judge_handles_not_found(self, judges_resource): + """get method propagates not found errors.""" + from layerlens._exceptions import NotFoundError + + mock_response = Mock() + mock_response.status_code = 404 + mock_response.headers = {} + + judges_resource._get.side_effect = NotFoundError("Judge not found", response=mock_response, body=None) + + with pytest.raises(NotFoundError): + judges_resource.get("nonexistent-judge") + + def test_create_judge_handles_bad_request(self, judges_resource): + """create method propagates bad request errors.""" + from layerlens._exceptions import BadRequestError + + mock_response = Mock() + mock_response.status_code = 400 + mock_response.headers = {} + + judges_resource._post.side_effect = BadRequestError("Invalid judge name", response=mock_response, body=None) + + with pytest.raises(BadRequestError): + judges_resource.create(name="ab", evaluation_goal="too short") + + def test_update_judge_handles_auth_error(self, judges_resource): + """update method propagates authentication errors.""" + from layerlens._exceptions import AuthenticationError + + mock_response = Mock() + mock_response.status_code = 401 + mock_response.headers = {} + + judges_resource._patch.side_effect = AuthenticationError("Unauthorized", response=mock_response, body=None) + + with pytest.raises(AuthenticationError): + judges_resource.update("judge-123", name="Updated") + + def test_delete_judge_handles_permission_error(self, judges_resource): + """delete method propagates permission errors.""" + from layerlens._exceptions import PermissionDeniedError + + mock_response = Mock() + mock_response.status_code = 403 + mock_response.headers = {} + + judges_resource._delete.side_effect = PermissionDeniedError("Access denied", response=mock_response, body=None) + + with pytest.raises(PermissionDeniedError): + judges_resource.delete("judge-123") + + +class TestJudgesURLConstruction: + """Test URL construction in Judges resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "custom-org" + client.project_id = "custom-proj" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def judges_resource(self, mock_client): + """Judges resource instance.""" + return Judges(mock_client) + + def test_base_url_construction(self, judges_resource): + """Base URL uses correct organization and project IDs.""" + assert judges_resource._base_url() == "/organizations/custom-org/projects/custom-proj/judges" + + def test_get_url_includes_judge_id(self, judges_resource): + """Get URL includes judge ID.""" + judges_resource._get.return_value = None + + judges_resource.get("judge-abc") + + call_args = judges_resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judges/judge-abc" + + def test_delete_url_includes_judge_id(self, judges_resource): + """Delete URL includes judge ID.""" + judges_resource._delete.return_value = None + + judges_resource.delete("judge-xyz") + + call_args = judges_resource._delete.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judges/judge-xyz" + + def test_update_url_includes_judge_id(self, judges_resource): + """Update URL includes judge ID.""" + judges_resource._patch.return_value = None + + judges_resource.update("judge-xyz", name="Updated") + + call_args = judges_resource._patch.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/judges/judge-xyz" diff --git a/tests/resources/test_models_resource.py b/tests/resources/test_models_resource.py index d9718b1..92eb7b4 100644 --- a/tests/resources/test_models_resource.py +++ b/tests/resources/test_models_resource.py @@ -13,7 +13,7 @@ class TestModels: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" @@ -374,7 +374,7 @@ class TestModelsErrorHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" @@ -442,7 +442,7 @@ class TestModelsTyping: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.organization_id = "org-123" client.project_id = "proj-456" diff --git a/tests/resources/test_results.py b/tests/resources/test_results.py index 99ec14f..c825c12 100644 --- a/tests/resources/test_results.py +++ b/tests/resources/test_results.py @@ -14,7 +14,7 @@ class TestResults: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.get_cast = Mock() return client @@ -272,7 +272,7 @@ class TestResultsErrorHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.get_cast = Mock() return client @@ -366,7 +366,7 @@ class TestResultsDataHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.get_cast = Mock() return client @@ -533,7 +533,7 @@ class TestResultsPagination: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.get_cast = Mock() return client @@ -717,7 +717,7 @@ class TestResultsPaginationErrorHandling: @pytest.fixture def mock_client(self): - """Mock Atlas client.""" + """Mock Stratix client.""" client = Mock() client.get_cast = Mock() return client diff --git a/tests/resources/test_trace_evaluations.py b/tests/resources/test_trace_evaluations.py new file mode 100644 index 0000000..b8e0295 --- /dev/null +++ b/tests/resources/test_trace_evaluations.py @@ -0,0 +1,444 @@ +from unittest.mock import Mock + +import pytest + +from layerlens.models import ( + TraceEvaluation, + CostEstimateResponse, + TraceEvaluationStatus, + TraceEvaluationsResponse, + TraceEvaluationResultsResponse, +) +from layerlens._constants import DEFAULT_TIMEOUT +from layerlens.resources.trace_evaluations.trace_evaluations import TraceEvaluations + + +class TestTraceEvaluations: + """Test TraceEvaluations resource API methods.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def trace_evals_resource(self, mock_client): + """TraceEvaluations resource instance.""" + return TraceEvaluations(mock_client) + + @pytest.fixture + def sample_trace_eval_data(self): + """Sample trace evaluation data.""" + return { + "id": "te-123", + "trace_id": "trace-456", + "judge_id": "judge-789", + "status": "success", + "judge_snapshot": { + "name": "Quality Judge", + "version": 1, + "evaluationGoal": "Evaluate output quality", + "modelId": "model-1", + "modelName": "GPT-4", + "modelCompany": "OpenAI", + }, + "created_at": "2024-01-01T00:00:00Z", + "started_at": "2024-01-01T00:00:01Z", + "finished_at": "2024-01-01T00:00:05Z", + } + + @pytest.fixture + def sample_result_data(self): + """Sample trace evaluation result data.""" + return { + "id": "result-123", + "trace_evaluation_id": "te-123", + "trace_id": "trace-456", + "judge_id": "judge-789", + "score": 0.85, + "passed": True, + "reasoning": "The output meets quality standards", + "steps": [ + {"step": 1, "reasoning": "Checked correctness"}, + {"step": 2, "reasoning": "Checked style"}, + ], + "model": "claude-sonnet-4-20250514", + "turns": 3, + "latency_ms": 2500, + "prompt_tokens": 1500, + "completion_tokens": 300, + "total_cost": 0.0045, + "created_at": "2024-01-01T00:00:05Z", + } + + def test_trace_evaluations_initialization(self, mock_client): + """TraceEvaluations resource initializes correctly.""" + te = TraceEvaluations(mock_client) + + assert te._client is mock_client + assert te._get is mock_client.get_cast + assert te._post is mock_client.post_cast + + def test_create_trace_evaluation_success(self, trace_evals_resource, sample_trace_eval_data): + """create method returns TraceEvaluation on success.""" + trace_evals_resource._post.return_value = sample_trace_eval_data + + result = trace_evals_resource.create(trace_id="trace-456", judge_id="judge-789") + + assert isinstance(result, TraceEvaluation) + assert result.id == "te-123" + assert result.trace_id == "trace-456" + assert result.judge_id == "judge-789" + assert result.status == TraceEvaluationStatus.SUCCESS + + def test_create_trace_evaluation_request_parameters(self, trace_evals_resource, sample_trace_eval_data): + """create method makes correct API request.""" + trace_evals_resource._post.return_value = sample_trace_eval_data + + trace_evals_resource.create(trace_id="trace-456", judge_id="judge-789") + + trace_evals_resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/trace-evaluations", + body={"trace_id": "trace-456", "judge_id": "judge-789"}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_create_trace_evaluation_none_response(self, trace_evals_resource): + """create method returns None when response is invalid.""" + trace_evals_resource._post.return_value = None + + result = trace_evals_resource.create(trace_id="trace-456", judge_id="judge-789") + + assert result is None + + def test_create_trace_evaluation_with_judge_snapshot(self, trace_evals_resource, sample_trace_eval_data): + """create method handles judge snapshot data.""" + trace_evals_resource._post.return_value = sample_trace_eval_data + + result = trace_evals_resource.create(trace_id="trace-456", judge_id="judge-789") + + assert result.judge_snapshot is not None + assert result.judge_snapshot.name == "Quality Judge" + assert result.judge_snapshot.version == 1 + assert result.judge_snapshot.evaluation_goal == "Evaluate output quality" + + def test_get_trace_evaluation_success(self, trace_evals_resource, sample_trace_eval_data): + """get method returns TraceEvaluation on success.""" + trace_evals_resource._get.return_value = sample_trace_eval_data + + result = trace_evals_resource.get("te-123") + + assert isinstance(result, TraceEvaluation) + assert result.id == "te-123" + + def test_get_trace_evaluation_request_parameters(self, trace_evals_resource, sample_trace_eval_data): + """get method makes correct API request.""" + trace_evals_resource._get.return_value = sample_trace_eval_data + + trace_evals_resource.get("te-123") + + trace_evals_resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/trace-evaluations/te-123", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_trace_evaluation_none_response(self, trace_evals_resource): + """get method returns None when response is invalid.""" + trace_evals_resource._get.return_value = None + + result = trace_evals_resource.get("te-123") + + assert result is None + + def test_get_many_trace_evaluations_success(self, trace_evals_resource, sample_trace_eval_data): + """get_many returns TraceEvaluationsResponse on success.""" + trace_evals_resource._get.return_value = { + "trace_evaluations": [sample_trace_eval_data], + "count": 1, + "total": 1, + } + + result = trace_evals_resource.get_many() + + assert isinstance(result, TraceEvaluationsResponse) + assert len(result.trace_evaluations) == 1 + assert result.trace_evaluations[0].id == "te-123" + assert result.count == 1 + assert result.total == 1 + + def test_get_many_with_filters(self, trace_evals_resource, sample_trace_eval_data): + """get_many passes filter parameters correctly.""" + trace_evals_resource._get.return_value = { + "trace_evaluations": [sample_trace_eval_data], + "count": 1, + "total": 1, + } + + trace_evals_resource.get_many( + page=2, + page_size=10, + judge_id="judge-789", + trace_id="trace-456", + outcome="pass", + time_range="7d", + search="quality", + sort_by="created_at", + sort_order="desc", + ) + + call_args = trace_evals_resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "2" + assert params["page_size"] == "10" + assert params["judgeId"] == "judge-789" + assert params["traceId"] == "trace-456" + assert params["outcome"] == "pass" + assert params["timeRange"] == "7d" + assert params["search"] == "quality" + assert params["sortBy"] == "created_at" + assert params["sortOrder"] == "desc" + + def test_get_many_default_pagination(self, trace_evals_resource): + """get_many uses default pagination.""" + trace_evals_resource._get.return_value = { + "trace_evaluations": [], + "count": 0, + "total": 0, + } + + trace_evals_resource.get_many() + + call_args = trace_evals_resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "1" + assert params["page_size"] == "20" + + def test_get_many_none_response(self, trace_evals_resource): + """get_many returns None when response is invalid.""" + trace_evals_resource._get.return_value = None + + result = trace_evals_resource.get_many() + + assert result is None + + def test_get_results_success(self, trace_evals_resource, sample_result_data): + """get_results returns TraceEvaluationResultsResponse on success.""" + trace_evals_resource._get.return_value = { + "results": [sample_result_data], + } + + result = trace_evals_resource.get_results("te-123") + + assert isinstance(result, TraceEvaluationResultsResponse) + assert len(result.results) == 1 + res = result.results[0] + assert res.id == "result-123" + assert res.score == 0.85 + assert res.passed is True + assert res.reasoning == "The output meets quality standards" + assert len(res.steps) == 2 + assert res.model == "claude-sonnet-4-20250514" + assert res.latency_ms == 2500 + + def test_get_results_request_parameters(self, trace_evals_resource, sample_result_data): + """get_results makes correct API request.""" + trace_evals_resource._get.return_value = {"results": [sample_result_data]} + + trace_evals_resource.get_results("te-123") + + trace_evals_resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/trace-evaluations/te-123/results", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_results_empty(self, trace_evals_resource): + """get_results returns empty results list.""" + trace_evals_resource._get.return_value = {"results": []} + + result = trace_evals_resource.get_results("te-123") + + assert isinstance(result, TraceEvaluationResultsResponse) + assert len(result.results) == 0 + + def test_get_results_none_response(self, trace_evals_resource): + """get_results returns None when response is invalid.""" + trace_evals_resource._get.return_value = None + + result = trace_evals_resource.get_results("te-123") + + assert result is None + + def test_estimate_cost_success(self, trace_evals_resource): + """estimate_cost returns CostEstimateResponse on success.""" + trace_evals_resource._post.return_value = { + "estimated_cost": 0.0045, + "input_tokens": 1500, + "output_tokens": 300, + "model": "claude-sonnet-4-20250514", + "trace_count": 5, + } + + result = trace_evals_resource.estimate_cost( + trace_ids=["t1", "t2", "t3", "t4", "t5"], + judge_id="judge-789", + ) + + assert isinstance(result, CostEstimateResponse) + assert result.estimated_cost == 0.0045 + assert result.trace_count == 5 + + def test_estimate_cost_request_parameters(self, trace_evals_resource): + """estimate_cost makes correct API request.""" + trace_evals_resource._post.return_value = { + "estimated_cost": 0.01, + "input_tokens": 100, + "output_tokens": 50, + "model": "test", + "trace_count": 2, + } + + trace_evals_resource.estimate_cost( + trace_ids=["t1", "t2"], + judge_id="judge-789", + ) + + trace_evals_resource._post.assert_called_once_with( + "/organizations/org-123/projects/proj-456/trace-evaluations/estimate", + body={"trace_ids": ["t1", "t2"], "judge_id": "judge-789"}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_estimate_cost_none_response(self, trace_evals_resource): + """estimate_cost returns None when response is invalid.""" + trace_evals_resource._post.return_value = None + + result = trace_evals_resource.estimate_cost(trace_ids=["t1"], judge_id="judge-789") + + assert result is None + + +class TestTraceEvaluationsErrorHandling: + """Test error handling in TraceEvaluations resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def trace_evals_resource(self, mock_client): + """TraceEvaluations resource instance.""" + return TraceEvaluations(mock_client) + + def test_create_handles_not_found(self, trace_evals_resource): + """create method propagates not found errors.""" + from layerlens._exceptions import NotFoundError + + mock_response = Mock() + mock_response.status_code = 404 + mock_response.headers = {} + + trace_evals_resource._post.side_effect = NotFoundError( + "Trace or judge not found", response=mock_response, body=None + ) + + with pytest.raises(NotFoundError): + trace_evals_resource.create(trace_id="bad-trace", judge_id="bad-judge") + + def test_get_handles_auth_error(self, trace_evals_resource): + """get method propagates authentication errors.""" + from layerlens._exceptions import AuthenticationError + + mock_response = Mock() + mock_response.status_code = 401 + mock_response.headers = {} + + trace_evals_resource._get.side_effect = AuthenticationError("Unauthorized", response=mock_response, body=None) + + with pytest.raises(AuthenticationError): + trace_evals_resource.get("te-123") + + def test_estimate_cost_handles_server_error(self, trace_evals_resource): + """estimate_cost method propagates server errors.""" + from layerlens._exceptions import InternalServerError + + mock_response = Mock() + mock_response.status_code = 500 + mock_response.headers = {} + + trace_evals_resource._post.side_effect = InternalServerError( + "Internal server error", response=mock_response, body=None + ) + + with pytest.raises(InternalServerError): + trace_evals_resource.estimate_cost(trace_ids=["t1"], judge_id="judge-789") + + +class TestTraceEvaluationsURLConstruction: + """Test URL construction in TraceEvaluations resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "custom-org" + client.project_id = "custom-proj" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def trace_evals_resource(self, mock_client): + """TraceEvaluations resource instance.""" + return TraceEvaluations(mock_client) + + def test_base_url_construction(self, trace_evals_resource): + """Base URL uses correct organization and project IDs.""" + assert trace_evals_resource._base_url() == "/organizations/custom-org/projects/custom-proj/trace-evaluations" + + def test_get_url_includes_evaluation_id(self, trace_evals_resource): + """Get URL includes evaluation ID.""" + trace_evals_resource._get.return_value = None + + trace_evals_resource.get("te-abc") + + call_args = trace_evals_resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/trace-evaluations/te-abc" + + def test_results_url_includes_evaluation_id(self, trace_evals_resource): + """Results URL includes evaluation ID.""" + trace_evals_resource._get.return_value = None + + trace_evals_resource.get_results("te-abc") + + call_args = trace_evals_resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/trace-evaluations/te-abc/results" + + def test_estimate_url(self, trace_evals_resource): + """Estimate URL is constructed correctly.""" + trace_evals_resource._post.return_value = None + + trace_evals_resource.estimate_cost(trace_ids=["t1"], judge_id="j1") + + call_args = trace_evals_resource._post.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/trace-evaluations/estimate" diff --git a/tests/resources/test_traces.py b/tests/resources/test_traces.py new file mode 100644 index 0000000..864f5ce --- /dev/null +++ b/tests/resources/test_traces.py @@ -0,0 +1,436 @@ +import os +import tempfile +from unittest.mock import Mock, patch + +import pytest + +from layerlens.models import Trace, TracesResponse, CreateTracesResponse +from layerlens._constants import DEFAULT_TIMEOUT +from layerlens.resources.traces.traces import Traces + + +class TestTraces: + """Test Traces resource API methods.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def traces_resource(self, mock_client): + """Traces resource instance.""" + return Traces(mock_client) + + @pytest.fixture + def sample_trace_data(self): + """Sample trace data for testing.""" + return { + "id": "trace-123", + "organization_id": "org-123", + "project_id": "proj-456", + "created_at": "2024-01-01T00:00:00Z", + "filename": "test-traces.json", + "data": {"input": "Hello", "output": "Hi there"}, + "input": "Hello", + } + + @pytest.fixture + def sample_trace_with_evals_data(self, sample_trace_data): + """Sample trace with evaluations data.""" + return { + **sample_trace_data, + "evaluations_count": 2, + "last_evaluations": [ + { + "judge_id": "judge-1", + "judge_name": "Quality Judge", + "judge_version": 1, + "created_at": "2024-01-02T00:00:00Z", + "passed": True, + }, + ], + } + + def test_traces_initialization(self, mock_client): + """Traces resource initializes correctly.""" + traces = Traces(mock_client) + + assert traces._client is mock_client + assert traces._get is mock_client.get_cast + assert traces._post is mock_client.post_cast + assert traces._delete is mock_client.delete_cast + + def test_get_trace_success(self, traces_resource, sample_trace_data): + """get method returns Trace on success.""" + traces_resource._get.return_value = sample_trace_data + + result = traces_resource.get("trace-123") + + assert isinstance(result, Trace) + assert result.id == "trace-123" + assert result.filename == "test-traces.json" + + def test_get_trace_request_parameters(self, traces_resource, sample_trace_data): + """get method makes correct API request.""" + traces_resource._get.return_value = sample_trace_data + + traces_resource.get("trace-123") + + traces_resource._get.assert_called_once_with( + "/organizations/org-123/projects/proj-456/traces/trace-123", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_get_trace_none_response(self, traces_resource): + """get method returns None when response is invalid.""" + traces_resource._get.return_value = None + + result = traces_resource.get("trace-123") + + assert result is None + + def test_get_many_traces_success(self, traces_resource, sample_trace_with_evals_data): + """get_many returns TracesResponse on success.""" + traces_resource._get.return_value = { + "traces": [sample_trace_with_evals_data], + "count": 1, + "total_count": 1, + } + + result = traces_resource.get_many() + + assert isinstance(result, TracesResponse) + assert len(result.traces) == 1 + assert result.traces[0].id == "trace-123" + assert result.traces[0].evaluations_count == 2 + assert result.count == 1 + + def test_get_many_traces_with_filters(self, traces_resource, sample_trace_with_evals_data): + """get_many passes filter parameters correctly.""" + traces_resource._get.return_value = { + "traces": [sample_trace_with_evals_data], + "count": 1, + "total_count": 1, + } + + traces_resource.get_many( + page=2, + page_size=50, + source="test-file.json", + judge_id="judge-1", + status="pass", + time_range="7d", + search="hello", + sort_by="created_at", + sort_order="desc", + ) + + call_args = traces_resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "2" + assert params["pageSize"] == "50" + assert params["source"] == "test-file.json" + assert params["judgeId"] == "judge-1" + assert params["status"] == "pass" + assert params["timeRange"] == "7d" + assert params["search"] == "hello" + assert params["sortBy"] == "created_at" + assert params["sortOrder"] == "desc" + + def test_get_many_traces_default_pagination(self, traces_resource): + """get_many uses default pagination.""" + traces_resource._get.return_value = { + "traces": [], + "count": 0, + "total_count": 0, + } + + traces_resource.get_many() + + call_args = traces_resource._get.call_args + params = call_args.kwargs["params"] + assert params["page"] == "1" + assert params["pageSize"] == "100" + + def test_get_many_traces_none_response(self, traces_resource): + """get_many returns None when response is invalid.""" + traces_resource._get.return_value = None + + result = traces_resource.get_many() + + assert result is None + + def test_delete_trace_success(self, traces_resource): + """delete method returns True on success.""" + traces_resource._delete.return_value = Mock() + + result = traces_resource.delete("trace-123") + + assert result is True + + def test_delete_trace_request_parameters(self, traces_resource): + """delete method makes correct API request.""" + traces_resource._delete.return_value = Mock() + + traces_resource.delete("trace-123") + + traces_resource._delete.assert_called_once_with( + "/organizations/org-123/projects/proj-456/traces/trace-123", + timeout=DEFAULT_TIMEOUT, + ) + + def test_delete_trace_none_response(self, traces_resource): + """delete method returns False when response is None.""" + traces_resource._delete.return_value = None + + result = traces_resource.delete("trace-123") + + assert result is False + + def test_get_sources_success(self, traces_resource): + """get_sources returns list of source filenames.""" + traces_resource._get.return_value = { + "sources": ["file1.json", "file2.jsonl", "imported-traces.json"], + } + + result = traces_resource.get_sources() + + assert result == ["file1.json", "file2.jsonl", "imported-traces.json"] + + def test_get_sources_empty(self, traces_resource): + """get_sources returns empty list when no sources.""" + traces_resource._get.return_value = {"sources": []} + + result = traces_resource.get_sources() + + assert result == [] + + def test_get_sources_invalid_response(self, traces_resource): + """get_sources returns empty list on invalid response.""" + traces_resource._get.return_value = None + + result = traces_resource.get_sources() + + assert result == [] + + +class TestTracesUpload: + """Test Traces upload functionality.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def traces_resource(self, mock_client): + """Traces resource instance.""" + return Traces(mock_client) + + def test_upload_json_file(self, traces_resource): + """upload method handles JSON files correctly.""" + with tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) as f: + f.write('[{"input": "hello", "output": "world"}]') + tmp_path = f.name + + try: + # Mock step 1: presigned URL (now returns dict) + traces_resource._post.side_effect = [ + {"url": "https://s3.example.com/upload?signed=true"}, + {"trace_ids": ["trace-new-1"]}, + ] + + # Mock step 2: S3 upload + with patch("layerlens.resources.traces.traces.httpx.put") as mock_put: + mock_put.return_value = Mock(status_code=200, raise_for_status=Mock()) + + result = traces_resource.upload(tmp_path) + + assert isinstance(result, CreateTracesResponse) + assert result.trace_ids == ["trace-new-1"] + + # Verify step 1 call + first_call = traces_resource._post.call_args_list[0] + assert first_call[0][0].endswith("/traces/upload") + body = first_call.kwargs["body"] + assert body["filename"] == os.path.basename(tmp_path) + assert body["type"] == "application/json" + + # Verify step 2 call + mock_put.assert_called_once() + put_args = mock_put.call_args + assert put_args[0][0] == "https://s3.example.com/upload?signed=true" + + # Verify step 3 call + third_call = traces_resource._post.call_args_list[1] + assert third_call[0][0].endswith("/traces") + assert third_call.kwargs["body"]["filename"] == os.path.basename(tmp_path) + finally: + os.unlink(tmp_path) + + def test_upload_jsonl_file(self, traces_resource): + """upload method handles JSONL files correctly.""" + with tempfile.NamedTemporaryFile(suffix=".jsonl", mode="w", delete=False) as f: + f.write('{"input": "hello"}\n{"input": "world"}\n') + tmp_path = f.name + + try: + traces_resource._post.side_effect = [ + {"url": "https://s3.example.com/upload"}, + {"trace_ids": ["trace-1", "trace-2"]}, + ] + + with patch("layerlens.resources.traces.traces.httpx.put") as mock_put: + mock_put.return_value = Mock(status_code=200, raise_for_status=Mock()) + + result = traces_resource.upload(tmp_path) + + assert isinstance(result, CreateTracesResponse) + assert len(result.trace_ids) == 2 + + first_call = traces_resource._post.call_args_list[0] + assert first_call.kwargs["body"]["type"] == "application/jsonl" + finally: + os.unlink(tmp_path) + + def test_upload_file_too_large(self, traces_resource): + """upload method raises ValueError for files exceeding 50 MB.""" + with tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) as f: + tmp_path = f.name + + try: + with patch("layerlens.resources.traces.traces.os.path.getsize", return_value=51 * 1024 * 1024): + with pytest.raises(ValueError, match="exceeds maximum"): + traces_resource.upload(tmp_path) + finally: + os.unlink(tmp_path) + + def test_upload_presigned_url_failure(self, traces_resource): + """upload method returns None when presigned URL request fails.""" + with tempfile.NamedTemporaryFile(suffix=".json", mode="w", delete=False) as f: + f.write("[]") + tmp_path = f.name + + try: + traces_resource._post.return_value = None + + result = traces_resource.upload(tmp_path) + + assert result is None + finally: + os.unlink(tmp_path) + + +class TestTracesErrorHandling: + """Test error handling in Traces resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def traces_resource(self, mock_client): + """Traces resource instance.""" + return Traces(mock_client) + + def test_get_trace_handles_not_found(self, traces_resource): + """get method propagates not found errors.""" + from layerlens._exceptions import NotFoundError + + mock_response = Mock() + mock_response.status_code = 404 + mock_response.headers = {} + + traces_resource._get.side_effect = NotFoundError("Trace not found", response=mock_response, body=None) + + with pytest.raises(NotFoundError): + traces_resource.get("nonexistent-trace") + + def test_delete_trace_handles_auth_error(self, traces_resource): + """delete method propagates authentication errors.""" + from layerlens._exceptions import AuthenticationError + + mock_response = Mock() + mock_response.status_code = 401 + mock_response.headers = {} + + traces_resource._delete.side_effect = AuthenticationError("Unauthorized", response=mock_response, body=None) + + with pytest.raises(AuthenticationError): + traces_resource.delete("trace-123") + + +class TestTracesURLConstruction: + """Test URL construction in Traces resource.""" + + @pytest.fixture + def mock_client(self): + """Mock Stratix client.""" + client = Mock() + client.organization_id = "custom-org" + client.project_id = "custom-proj" + client.get_cast = Mock() + client.post_cast = Mock() + client.patch_cast = Mock() + client.delete_cast = Mock() + return client + + @pytest.fixture + def traces_resource(self, mock_client): + """Traces resource instance.""" + return Traces(mock_client) + + def test_base_url_construction(self, traces_resource): + """Base URL uses correct organization and project IDs.""" + assert traces_resource._base_url() == "/organizations/custom-org/projects/custom-proj/traces" + + def test_get_url_includes_trace_id(self, traces_resource): + """Get URL includes trace ID.""" + traces_resource._get.return_value = None + + traces_resource.get("trace-abc") + + call_args = traces_resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/traces/trace-abc" + + def test_delete_url_includes_trace_id(self, traces_resource): + """Delete URL includes trace ID.""" + traces_resource._delete.return_value = None + + traces_resource.delete("trace-xyz") + + call_args = traces_resource._delete.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/traces/trace-xyz" + + def test_sources_url(self, traces_resource): + """Sources URL is constructed correctly.""" + traces_resource._get.return_value = {"sources": []} + + traces_resource.get_sources() + + call_args = traces_resource._get.call_args + assert call_args[0][0] == "/organizations/custom-org/projects/custom-proj/traces/sources" diff --git a/tests/test_client.py b/tests/test_client.py index 558494a..4dfc1b7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2,12 +2,12 @@ import pytest -from layerlens import Atlas -from layerlens._exceptions import AtlasError +from layerlens import Stratix +from layerlens._exceptions import StratixError -class TestAtlasClientInitialization: - """Test Atlas client initialization and configuration.""" +class TestStratixClientInitialization: + """Test Stratix client initialization and configuration.""" @pytest.fixture def mock_org(self): @@ -19,16 +19,16 @@ def mock_org(self): def test_none_values_fallback_to_env(self, mock_env_vars, mock_org): """None values explicitly passed fallback to environment.""" _ = mock_env_vars # Fixture used for side effects - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key=None) + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key=None) assert client.api_key == "test-api-key" @pytest.mark.parametrize("base_url", ["https://custom.api.com", "https://staging.layerlens.ai/api/v1"]) def test_custom_base_url(self, base_url, mock_org): """Client accepts custom base URL.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key", base_url=base_url) + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key", base_url=base_url) assert str(client.base_url).rstrip("/") == base_url.rstrip("/") @@ -36,15 +36,15 @@ def test_custom_timeout(self, mock_org): """Client accepts custom timeout.""" import httpx - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key", timeout=30.0) + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key", timeout=30.0) assert isinstance(client.timeout, httpx.Timeout) def test_auth_headers_with_api_key(self, mock_org): """auth_headers property returns correct headers when API key is set.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-api-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-api-key") headers = client.auth_headers @@ -52,32 +52,32 @@ def test_auth_headers_with_api_key(self, mock_org): def test_auth_headers_without_api_key(self, mock_org): """auth_headers property returns empty dict when no API key.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): with pytest.raises( - AtlasError, - match="The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable", + StratixError, + match="The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_STRATIX_API_KEY environment variable", ): - Atlas(api_key="") + Stratix(api_key="") def test_auth_headers_with_empty_api_key(self, mock_org): """auth_headers property returns empty dict when API key is empty string.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): with pytest.raises( - AtlasError, - match="The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_ATLAS_API_KEY environment variable", + StratixError, + match="The api_key client option must be set either by passing api_key to the client or by setting the LAYERLENS_STRATIX_API_KEY environment variable", ): - Atlas(api_key="") + Stratix(api_key="") def test_copy_method(self, mock_org): """copy method creates new client with overridden parameters.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - original_client = Atlas( + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + original_client = Stratix( api_key="original-key", base_url="https://original.api.com", timeout=10.0, ) - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): new_client = original_client.copy(api_key="new-key", timeout=20.0) # Check overridden values @@ -91,20 +91,20 @@ def test_copy_method(self, mock_org): def test_copy_method_partial_override(self, mock_org): """copy method allows partial parameter override.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - original_client = Atlas(api_key="original-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + original_client = Stratix(api_key="original-key") - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): new_client = original_client.copy(api_key="new-key") assert new_client.api_key == "new-key" def test_with_options_alias(self, mock_org): """with_options is an alias for copy method.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - original_client = Atlas(api_key="original-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + original_client = Stratix(api_key="original-key") - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): new_client = original_client.with_options(api_key="new-key") assert new_client.api_key == "new-key" @@ -113,10 +113,10 @@ def test_with_options_alias(self, mock_org): def test_copy_method_timeout_override(self, mock_org): """copy method properly overrides timeout when original is None.""" # Create a client with no explicit timeout (uses default) - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - original_client = Atlas(api_key="original-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + original_client = Stratix(api_key="original-key") - with patch("layerlens.Atlas._get_organization", return_value=mock_org): + with patch("layerlens.Stratix._get_organization", return_value=mock_org): new_client = original_client.copy(timeout=30.0) import httpx @@ -126,8 +126,8 @@ def test_copy_method_timeout_override(self, mock_org): assert new_client.timeout == original_client.timeout -class TestAtlasClientErrorHandling: - """Test error handling in Atlas client.""" +class TestStratixClientErrorHandling: + """Test error handling in Stratix client.""" @pytest.fixture def mock_org(self): @@ -149,8 +149,8 @@ def test_make_status_error_bad_request(self, mock_org): """_make_status_error creates BadRequestError for 400 status.""" from layerlens._exceptions import BadRequestError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(400) mock_body = {"error": "Bad request"} @@ -163,8 +163,8 @@ def test_make_status_error_unauthorized(self, mock_org): """_make_status_error creates AuthenticationError for 401 status.""" from layerlens._exceptions import AuthenticationError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(401) mock_body = {"error": "Unauthorized"} @@ -177,8 +177,8 @@ def test_make_status_error_forbidden(self, mock_org): """_make_status_error creates PermissionDeniedError for 403 status.""" from layerlens._exceptions import PermissionDeniedError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(403) mock_body = {"error": "Forbidden"} @@ -191,8 +191,8 @@ def test_make_status_error_not_found(self, mock_org): """_make_status_error creates NotFoundError for 404 status.""" from layerlens._exceptions import NotFoundError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(404) mock_body = {"error": "Not found"} @@ -205,8 +205,8 @@ def test_make_status_error_conflict(self, mock_org): """_make_status_error creates ConflictError for 409 status.""" from layerlens._exceptions import ConflictError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(409) mock_body = {"error": "Conflict"} @@ -219,8 +219,8 @@ def test_make_status_error_unprocessable_entity(self, mock_org): """_make_status_error creates UnprocessableEntityError for 422 status.""" from layerlens._exceptions import UnprocessableEntityError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(422) mock_body = {"error": "Unprocessable entity"} @@ -233,8 +233,8 @@ def test_make_status_error_rate_limit(self, mock_org): """_make_status_error creates RateLimitError for 429 status.""" from layerlens._exceptions import RateLimitError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(429) mock_body = {"error": "Rate limited"} @@ -247,8 +247,8 @@ def test_make_status_error_internal_server_error(self, mock_org): """_make_status_error creates InternalServerError for 500+ status.""" from layerlens._exceptions import InternalServerError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(500) mock_body = {"error": "Internal server error"} @@ -261,8 +261,8 @@ def test_make_status_error_gateway_timeout(self, mock_org): """_make_status_error creates InternalServerError for 502 status.""" from layerlens._exceptions import InternalServerError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(502) mock_body = {"error": "Gateway timeout"} @@ -275,8 +275,8 @@ def test_make_status_error_unknown_status(self, mock_org): """_make_status_error creates generic APIStatusError for unknown status codes.""" from layerlens._exceptions import APIStatusError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(418) # I'm a teapot mock_body = {"error": "Unknown error"} @@ -289,8 +289,8 @@ def test_make_status_error_with_non_mapping_body(self, mock_org): """_make_status_error handles non-mapping body correctly.""" from layerlens._exceptions import NotFoundError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(404) mock_body = "Simple string error" @@ -303,8 +303,8 @@ def test_make_status_error_with_none_body(self, mock_org): """_make_status_error handles None body correctly.""" from layerlens._exceptions import BadRequestError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(400) error = client._make_status_error("Bad request", body=None, response=mock_response) @@ -316,8 +316,8 @@ def test_make_status_error_with_complex_body(self, mock_org): """_make_status_error extracts error from complex body structure.""" from layerlens._exceptions import AuthenticationError - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="test-key") mock_response = self._create_mock_response(401) mock_body = { "error": {"message": "Invalid API key", "code": "AUTH_ERROR"}, diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 5a34150..0d5eece 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -6,7 +6,7 @@ from layerlens._exceptions import ( APIError, - AtlasError, + StratixError, ConflictError, NotFoundError, APIStatusError, @@ -25,19 +25,19 @@ class TestExceptionHierarchy: """Test exception inheritance and basic functionality.""" - def test_atlas_error_is_base_exception(self): - """AtlasError inherits from Exception.""" - error = AtlasError("test message") + def test_stratix_error_is_base_exception(self): + """StratixError inherits from Exception.""" + error = StratixError("test message") assert isinstance(error, Exception) assert str(error) == "test message" - def test_api_error_inherits_from_atlas_error(self): - """APIError inherits from AtlasError.""" + def test_api_error_inherits_from_stratix_error(self): + """APIError inherits from StratixError.""" mock_request = Mock(spec=httpx.Request) error = APIError("api error", mock_request, body=None) - assert isinstance(error, AtlasError) + assert isinstance(error, StratixError) assert isinstance(error, Exception) def test_api_status_error_inherits_from_api_error(self): @@ -50,7 +50,7 @@ def test_api_status_error_inherits_from_api_error(self): error = APIStatusError("status error", response=mock_response, body=None) assert isinstance(error, APIError) - assert isinstance(error, AtlasError) + assert isinstance(error, StratixError) @pytest.mark.parametrize( "exception_class", @@ -76,7 +76,7 @@ def test_status_exceptions_inherit_from_api_status_error(self, exception_class): assert isinstance(error, APIStatusError) assert isinstance(error, APIError) - assert isinstance(error, AtlasError) + assert isinstance(error, StratixError) class TestAPIError: diff --git a/tests/test_integration.py b/tests/test_integration.py index 71eb122..bd05641 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -4,7 +4,7 @@ import httpx import pytest -from layerlens import Atlas +from layerlens import Stratix from layerlens.models import ( Model, Result, @@ -15,13 +15,13 @@ ) -class TestAtlasIntegration: - """Integration tests for full Atlas API workflows.""" +class TestStratixIntegration: + """Integration tests for full Stratix API workflows.""" @pytest.fixture - def atlas_client(self): - """Create Atlas client with mocked dependencies.""" - return Atlas(api_key="test-api-key") + def stratix_client(self): + """Create Stratix client with mocked dependencies.""" + return Stratix(api_key="test-api-key") @pytest.fixture def sample_model_data(self): @@ -98,16 +98,16 @@ class TestCompleteEvaluationWorkflow: """Test complete evaluation workflow from start to finish.""" @pytest.fixture - def atlas_client(self): - """Atlas client for workflow testing.""" + def stratix_client(self): + """Stratix client for workflow testing.""" mock_org = Mock() mock_org.id = "org-123" mock_org.projects = [Mock(id="proj-456")] - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - return Atlas(api_key="workflow-test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + return Stratix(api_key="workflow-test-key") - def test_complete_evaluation_workflow(self, atlas_client): + def test_complete_evaluation_workflow(self, stratix_client): """Test complete workflow: get models/benchmarks -> create evaluation -> get results.""" # Mock data @@ -174,7 +174,9 @@ def test_complete_evaluation_workflow(self, atlas_client): # Mock responses evaluations_response = CreateEvaluationsResponse(data=[evaluation]) - with patch.object(atlas_client, "get_cast") as mock_get, patch.object(atlas_client, "post_cast") as mock_post: + with patch.object(stratix_client, "get_cast") as mock_get, patch.object( + stratix_client, "post_cast" + ) as mock_post: # Configure mocks for the workflow mock_get.return_value = { "evaluation_id": "eval-789", @@ -189,13 +191,13 @@ def test_complete_evaluation_workflow(self, atlas_client): } # Get results - raw API response mock_post.return_value = evaluations_response # Create evaluation - # Step 1: Create evaluation directly (Atlas client doesn't expose models/benchmarks resources) - created_evaluation = atlas_client.evaluations.create(model=model, benchmark=benchmark) + # Step 1: Create evaluation directly (Stratix client doesn't expose models/benchmarks resources) + created_evaluation = stratix_client.evaluations.create(model=model, benchmark=benchmark) assert created_evaluation.id == "eval-789" assert created_evaluation.status == EvaluationStatus.SUCCESS # Step 2: Get evaluation results - results = atlas_client.results.get(evaluation=created_evaluation) + results = stratix_client.results.get(evaluation=created_evaluation) assert len(results.results) == 1 assert results.results[0].score == 1.0 assert results.results[0].subset == "math" @@ -211,7 +213,7 @@ def test_complete_evaluation_workflow(self, atlas_client): post_call = mock_post.call_args_list[0] assert "/evaluations" in post_call[0][0] - def test_workflow_with_error_handling(self, atlas_client): + def test_workflow_with_error_handling(self, stratix_client): """Test workflow handles errors gracefully.""" from layerlens._exceptions import NotFoundError @@ -219,16 +221,16 @@ def test_workflow_with_error_handling(self, atlas_client): mock_response.status_code = 404 mock_response.headers = {} - with patch.object(atlas_client, "get_cast") as mock_get: + with patch.object(stratix_client, "get_cast") as mock_get: # Mock API error when getting results api_error = NotFoundError("Results not found", response=mock_response, body=None) mock_get.side_effect = api_error # Verify error is propagated with pytest.raises(NotFoundError): - atlas_client.results.get_by_id(evaluation_id="test-eval") + stratix_client.results.get_by_id(evaluation_id="test-eval") - def test_workflow_with_custom_timeouts(self, atlas_client): + def test_workflow_with_custom_timeouts(self, stratix_client): """Test workflow respects custom timeout settings.""" result_data = { "subset": "test", @@ -240,7 +242,7 @@ def test_workflow_with_custom_timeouts(self, atlas_client): "metrics": {"accuracy": 1.0}, } - with patch.object(atlas_client, "get_cast") as mock_get: + with patch.object(stratix_client, "get_cast") as mock_get: mock_get.return_value = { "evaluation_id": "test-eval", "results": [result_data], @@ -255,7 +257,7 @@ def test_workflow_with_custom_timeouts(self, atlas_client): # Test with custom timeout custom_timeout = httpx.Timeout(30.0) - results = atlas_client.results.get_by_id(evaluation_id="test-eval", timeout=custom_timeout) + results = stratix_client.results.get_by_id(evaluation_id="test-eval", timeout=custom_timeout) assert len(results.results) == 1 @@ -268,16 +270,16 @@ class TestResourceInteraction: """Test interactions between different resources.""" @pytest.fixture - def atlas_client(self): - """Atlas client for resource interaction testing.""" + def stratix_client(self): + """Stratix client for resource interaction testing.""" mock_org = Mock() mock_org.id = "org-123" mock_org.projects = [Mock(id="proj-456")] - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - return Atlas(api_key="interaction-test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + return Stratix(api_key="interaction-test-key") - def test_evaluation_creation_with_model_and_benchmark_objects(self, atlas_client): + def test_evaluation_creation_with_model_and_benchmark_objects(self, stratix_client): """Test creating evaluation using model and benchmark objects.""" # Create model and benchmark objects @@ -331,11 +333,11 @@ def test_evaluation_creation_with_model_and_benchmark_objects(self, atlas_client evaluations_response = CreateEvaluationsResponse(data=[evaluation]) - with patch.object(atlas_client, "post_cast") as mock_post: + with patch.object(stratix_client, "post_cast") as mock_post: mock_post.return_value = evaluations_response # Create evaluation using model and benchmark keys - created_evaluation = atlas_client.evaluations.create(model=model, benchmark=benchmark) + created_evaluation = stratix_client.evaluations.create(model=model, benchmark=benchmark) assert created_evaluation.id == "eval-interaction" assert created_evaluation.model_id == model.id @@ -347,7 +349,7 @@ def test_evaluation_creation_with_model_and_benchmark_objects(self, atlas_client assert body["model_id"] == model.id assert body["dataset_id"] == benchmark.id - def test_results_analysis_workflow(self, atlas_client): + def test_results_analysis_workflow(self, stratix_client): """Test analyzing results from multiple evaluations.""" # Create multiple result objects @@ -383,7 +385,7 @@ def test_results_analysis_workflow(self, atlas_client): results = [Result(**data) for data in results_data] - with patch.object(atlas_client, "get_cast") as mock_get: + with patch.object(stratix_client, "get_cast") as mock_get: mock_get.return_value = { "evaluation_id": "test-eval", "results": results_data, @@ -397,7 +399,7 @@ def test_results_analysis_workflow(self, atlas_client): } # Get results - evaluation_results = atlas_client.results.get_by_id(evaluation_id="test-eval") + evaluation_results = stratix_client.results.get_by_id(evaluation_id="test-eval") # Analyze results math_results = [r for r in evaluation_results.results if r.subset == "math"] @@ -421,8 +423,8 @@ def test_results_analysis_workflow(self, atlas_client): assert abs(avg_duration - expected_avg) < 0.01 -class TestAtlasClientProperties: - """Test Atlas client resource properties and access.""" +class TestStratixClientProperties: + """Test Stratix client resource properties and access.""" @pytest.fixture def mock_org(self): @@ -432,9 +434,9 @@ def mock_org(self): return org def test_client_has_all_resource_properties(self, mock_org): - """Atlas client exposes all resource properties.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="property-test-key") + """Stratix client exposes all resource properties.""" + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="property-test-key") # Verify available resource properties exist assert hasattr(client, "evaluations") @@ -449,8 +451,8 @@ def test_client_has_all_resource_properties(self, mock_org): def test_resource_properties_share_same_client(self, mock_org): """All resource properties share the same client instance.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="shared-client-test") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="shared-client-test") # Verify all resources use the same client assert client.evaluations._client is client @@ -474,14 +476,14 @@ def mock_org2(self): org.projects = [Mock(id="proj-123")] return org - def test_multiple_atlas_clients_independent(self, mock_org1, mock_org2): - """Multiple Atlas client instances operate independently.""" + def test_multiple_stratix_clients_independent(self, mock_org1, mock_org2): + """Multiple Stratix client instances operate independently.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org1): - client1 = Atlas(api_key="client-1-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org1): + client1 = Stratix(api_key="client-1-key") - with patch("layerlens.Atlas._get_organization", return_value=mock_org2): - client2 = Atlas(api_key="client-2-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org2): + client2 = Stratix(api_key="client-2-key") # Verify clients are independent assert client1.api_key != client2.api_key @@ -493,11 +495,11 @@ def test_multiple_atlas_clients_independent(self, mock_org1, mock_org2): def test_resource_operations_isolated(self, mock_org1, mock_org2): """Operations on different client resources are isolated.""" - with patch("layerlens.Atlas._get_organization", return_value=mock_org1): - client1 = Atlas(api_key="iso-test-1") + with patch("layerlens.Stratix._get_organization", return_value=mock_org1): + client1 = Stratix(api_key="iso-test-1") - with patch("layerlens.Atlas._get_organization", return_value=mock_org2): - client2 = Atlas(api_key="iso-test-2") + with patch("layerlens.Stratix._get_organization", return_value=mock_org2): + client2 = Stratix(api_key="iso-test-2") result_data = { "subset": "test", @@ -601,8 +603,8 @@ def test_evaluation_workflow_error_propagation(self, mock_org): model = Model(**model_data) benchmark = Benchmark(**benchmark_data) - with patch("layerlens.Atlas._get_organization", return_value=mock_org): - client = Atlas(api_key="error-test-key") + with patch("layerlens.Stratix._get_organization", return_value=mock_org): + client = Stratix(api_key="error-test-key") mock_response = Mock() mock_response.status_code = 500 diff --git a/tests/test_resource.py b/tests/test_resource.py index 7502c54..ce26fcb 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -10,7 +10,7 @@ class TestSyncAPIResource: @pytest.fixture def mock_client(self): - """Mock Atlas client with required methods.""" + """Mock Stratix client with required methods.""" client = Mock() client.get_cast = Mock() client.post_cast = Mock() @@ -292,7 +292,7 @@ class TestSyncAPIResourceTyping: def test_resource_client_attribute_typing(self): """Resource._client maintains proper typing.""" - # Test with properly typed client (would be Atlas in real usage) + # Test with properly typed client (would be Stratix in real usage) mock_client = Mock() mock_client.get_cast = Mock() mock_client.post_cast = Mock() diff --git a/tests/test_utils.py b/tests/test_utils.py index 1be7f06..6d93a20 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -172,7 +172,7 @@ def test_filter_redacts_sensitive_headers(self, filter_instance, mock_log_record "content-type": "application/json", "x-api-key": "secret-key-123", "authorization": "Bearer token-456", - "user-agent": "atlas-python-sdk", + "user-agent": "stratix-python-sdk", } } @@ -183,7 +183,7 @@ def test_filter_redacts_sensitive_headers(self, filter_instance, mock_log_record assert headers["content-type"] == "application/json" assert headers["x-api-key"] == "" assert headers["authorization"] == "" - assert headers["user-agent"] == "atlas-python-sdk" + assert headers["user-agent"] == "stratix-python-sdk" def test_filter_handles_case_insensitive_headers(self, filter_instance, mock_log_record): """filter redacts headers regardless of case."""