diff --git a/docs/api-reference/models-benchmarks.md b/docs/api-reference/models-benchmarks.md index bef109ac..aeffd32e 100644 --- a/docs/api-reference/models-benchmarks.md +++ b/docs/api-reference/models-benchmarks.md @@ -101,7 +101,7 @@ Returns an `Optional[Model]` - a single `Model` object if found, or `None` if th ### `add(*model_ids, timeout=None)` -Adds public models to the project by their IDs. +Adds models (public or custom) to the project by their IDs. #### Parameters @@ -123,7 +123,7 @@ success = client.models.add("model-id-1", "model-id-2") ### `remove(*model_ids, timeout=None)` -Removes models from the project by their IDs. +Removes models (public or custom) from the project's model list. The underlying records are not deleted — use `delete_custom` to fully tear down a custom model. #### Parameters @@ -187,6 +187,58 @@ if result: print(f"Created model: {result.model_id}") ``` +### `update_custom(model_id, *, api_url=None, api_key=None, max_tokens=None, timeout=None)` + +Updates a custom model's mutable fields. At least one of `api_url`, `api_key`, or `max_tokens` must be provided. Primary use case: repointing `api_url` for ephemeral vLLM endpoints behind cloudflared tunnels whose URL changes between sessions. + +#### Parameters + +| Parameter | Type | Required | Description | +| ------------ | -------------------------------- | -------- | -------------------------------------------------------- | +| `model_id` | `str` | Yes | ID of the custom model to update | +| `api_url` | `str \| None` | No | New base URL for the OpenAI-compatible API endpoint | +| `api_key` | `str \| None` | No | New API key for the model provider | +| `max_tokens` | `int \| None` | No | New maximum tokens value | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns `bool` — `True` on success, `False` otherwise. + +#### Example + +```python +client = Stratix() + +# Repoint the api_url without re-creating the model +client.models.update_custom( + "model-id-from-create-custom", + api_url="https://my-new-endpoint.example.com/v1", +) +``` + +### `delete_custom(model_id, *, timeout=None)` + +Disables a custom model and removes it from `Project.Models`. The backend tears down the model's S3 yaml artifacts and AWS secret, and marks the record as disabled (preserving any evaluation references). Public models cannot be deleted via the SDK. + +#### Parameters + +| Parameter | Type | Required | Description | +| ---------- | -------------------------------- | -------- | --------------------------------- | +| `model_id` | `str` | Yes | ID of the custom model to delete | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns `bool` — `True` on success, `False` otherwise. + +#### Example + +```python +client = Stratix() +client.models.delete_custom("model-id-from-create-custom") +``` + ## Benchmarks ### `get(type=None, name=None, key=None, categories=None, languages=None, timeout=None)` diff --git a/docs/examples/models-and-benchmarks.md b/docs/examples/models-and-benchmarks.md index 67d1d7fe..e517344f 100644 --- a/docs/examples/models-and-benchmarks.md +++ b/docs/examples/models-and-benchmarks.md @@ -133,6 +133,69 @@ def main(): print(f" - {m.name} (id={m.id}, key={m.key})") +if __name__ == "__main__": + main() +``` + +## Repointing a Custom Model's `api_url` + +Use this when your model's endpoint URL changes — for example, when serving a vLLM instance behind a cloudflared tunnel that rotates its hostname between sessions. + +```python +from layerlens import Stratix + + +def main(): + client = Stratix() + + result = client.models.create_custom( + name="My Tunnel-backed Model", + key="my-org/tunnel-model-v1", + description="vLLM served behind a cloudflared tunnel", + api_url="https://tunnel-1.example.com/v1", + api_key="my-provider-api-key", + max_tokens=4096, + ) + assert result is not None + + # Later, when the tunnel URL changes: + client.models.update_custom( + result.model_id, + api_url="https://tunnel-2.example.com/v1", + ) + + # Run evaluations as usual — the model now points at the new endpoint. + + +if __name__ == "__main__": + main() +``` + +## Replacing a Custom Model + +`delete_custom` releases the model's name so it can be reused. This is useful for replacing a misconfigured model without picking a new name. + +```python +from layerlens import Stratix + + +def main(): + client = Stratix() + + # Tear down the old version + client.models.delete_custom("old-model-id") + + # Recreate with the same name (now free) + client.models.create_custom( + name="My Custom Model", + key="my-org/custom-model-v2", + description="Replacement after schema migration", + api_url="https://my-endpoint.example.com/v1", + api_key="my-provider-api-key", + max_tokens=4096, + ) + + if __name__ == "__main__": main() ``` diff --git a/samples/core/custom_model.py b/samples/core/custom_model.py index 327e6d64..e660fb04 100644 --- a/samples/core/custom_model.py +++ b/samples/core/custom_model.py @@ -58,6 +58,32 @@ def main() -> None: else: print("\nNo custom models found in project") + if not result: + return + + # ── Update mutable fields (e.g. repoint api_url) ────────────────── + # + # Use this when your endpoint URL changes -- common for vLLM + # instances served behind cloudflared tunnels whose hostname + # rotates between sessions. + + updated = client.models.update_custom( + result.model_id, + api_url="https://my-new-endpoint.example.com/v1", + ) + if updated: + print(f"\nCustom model {result.model_id} api_url updated") + + # ── Full teardown ───────────────────────────────────────────────── + # + # ``delete_custom`` disables the model, removes it from + # ``Project.Models``, and releases its name for reuse. Evaluation + # references to the disabled record are preserved. + + deleted = client.models.delete_custom(result.model_id) + if deleted: + print(f"Custom model {result.model_id} deleted") + if __name__ == "__main__": main() diff --git a/src/layerlens/_version.py b/src/layerlens/_version.py index a2fe4aa3..fb6b8f67 100644 --- a/src/layerlens/_version.py +++ b/src/layerlens/_version.py @@ -1,4 +1,4 @@ -__version__ = "1.6.0" +__version__ = "1.6.1" # Will be templated during the build __git_commit__ = "__GIT_COMMIT__" diff --git a/src/layerlens/resources/models/models.py b/src/layerlens/resources/models/models.py index 627b60fe..30ad5579 100644 --- a/src/layerlens/resources/models/models.py +++ b/src/layerlens/resources/models/models.py @@ -169,10 +169,10 @@ def add( *model_ids: str, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> bool: - """Add models to the project by their IDs.""" - # Only fetch public (platform) models — custom models are managed - # separately and must not be included in the project patch payload. - current = self.get(timeout=timeout, type="public") or [] + """Add models (public or custom) to the project by their IDs.""" + # Fetch the full current list (public + custom). The project's + # PATCH endpoint expects the complete model set in a single payload. + current = self.get(timeout=timeout) or [] current_ids = [str(m.id) for m in current] new_ids = list(dict.fromkeys(current_ids + list(model_ids))) return self._patch_project_models(new_ids, timeout) @@ -182,10 +182,13 @@ def remove( *model_ids: str, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> bool: - """Remove models from the project by their IDs.""" - # Only fetch public (platform) models — custom models are managed - # separately and must not be included in the project patch payload. - current = self.get(timeout=timeout, type="public") or [] + """Remove models (public or custom) from the project's model list. + + Note: this only detaches the models from the project. The underlying + records are not deleted — use ``delete_custom`` to fully tear down a + custom model. + """ + current = self.get(timeout=timeout) or [] remove_set = set(model_ids) new_ids = [str(m.id) for m in current if str(m.id) not in remove_set] return self._patch_project_models(new_ids, timeout) @@ -256,6 +259,61 @@ def create_custom( return CreateModelResponse(**resp) return None + def update_custom( + self, + model_id: str, + *, + api_url: Optional[str] = None, + api_key: Optional[str] = None, + max_tokens: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + """Update a custom model's mutable fields. + + At least one of ``api_url``, ``api_key``, or ``max_tokens`` must be + provided. Returns ``True`` on success. + + Primary use case: repointing ``api_url`` for ephemeral vLLM endpoints + behind cloudflared tunnels whose URL changes between sessions. + + Args: + model_id: ID of the custom model to update. + api_url: New base URL for the OpenAI-compatible API endpoint. + api_key: New API key for the model provider. + max_tokens: New maximum tokens value. + timeout: Request timeout override. + """ + url = ( + f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/custom-models/{model_id}" + ) + body: Dict[str, Any] = {} + if api_url is not None: + body["api_url"] = api_url + if api_key is not None: + body["api_key"] = api_key + if max_tokens is not None: + body["max_tokens"] = max_tokens + resp = self._patch(url, body=body, timeout=timeout, cast_to=dict) + return isinstance(resp, dict) and "data" in resp + + def delete_custom( + self, + model_id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + """Disable a custom model and detach it from the project. + + The backend tears down the model's S3 yaml artifacts and the AWS + secret, marks the record as disabled (preserving evaluation + references), and removes the model ID from ``Project.Models``. + """ + url = ( + f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/custom-models/{model_id}" + ) + resp = self._delete(url, timeout=timeout, cast_to=dict) + return isinstance(resp, dict) and "data" in resp + class AsyncModels(AsyncAPIResource): async def get( @@ -368,10 +426,10 @@ async def add( *model_ids: str, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> bool: - """Add models to the project by their IDs.""" - # Only fetch public (platform) models — custom models are managed - # separately and must not be included in the project patch payload. - current = await self.get(timeout=timeout, type="public") or [] + """Add models (public or custom) to the project by their IDs.""" + # Fetch the full current list (public + custom). The project's + # PATCH endpoint expects the complete model set in a single payload. + current = await self.get(timeout=timeout) or [] current_ids = [str(m.id) for m in current] new_ids = list(dict.fromkeys(current_ids + list(model_ids))) return await self._patch_project_models(new_ids, timeout) @@ -381,10 +439,13 @@ async def remove( *model_ids: str, timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, ) -> bool: - """Remove models from the project by their IDs.""" - # Only fetch public (platform) models — custom models are managed - # separately and must not be included in the project patch payload. - current = await self.get(timeout=timeout, type="public") or [] + """Remove models (public or custom) from the project's model list. + + Note: this only detaches the models from the project. The underlying + records are not deleted — use ``delete_custom`` to fully tear down a + custom model. + """ + current = await self.get(timeout=timeout) or [] remove_set = set(model_ids) new_ids = [str(m.id) for m in current if str(m.id) not in remove_set] return await self._patch_project_models(new_ids, timeout) @@ -454,3 +515,43 @@ async def create_custom( if isinstance(resp, dict) and "model_id" in resp: return CreateModelResponse(**resp) return None + + async def update_custom( + self, + model_id: str, + *, + api_url: Optional[str] = None, + api_key: Optional[str] = None, + max_tokens: Optional[int] = None, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + """Update a custom model's mutable fields. + + At least one of ``api_url``, ``api_key``, or ``max_tokens`` must be + provided. Returns ``True`` on success. + """ + url = ( + f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/custom-models/{model_id}" + ) + body: Dict[str, Any] = {} + if api_url is not None: + body["api_url"] = api_url + if api_key is not None: + body["api_key"] = api_key + if max_tokens is not None: + body["max_tokens"] = max_tokens + resp = await self._patch(url, body=body, timeout=timeout, cast_to=dict) + return isinstance(resp, dict) and "data" in resp + + async def delete_custom( + self, + model_id: str, + *, + timeout: float | httpx.Timeout | None = DEFAULT_TIMEOUT, + ) -> bool: + """Disable a custom model and detach it from the project.""" + url = ( + f"/organizations/{self._client.organization_id}/projects/{self._client.project_id}/custom-models/{model_id}" + ) + resp = await self._delete(url, timeout=timeout, cast_to=dict) + return isinstance(resp, dict) and "data" in resp diff --git a/tests/resources/test_models_resource.py b/tests/resources/test_models_resource.py index 085177b5..6fbd4895 100644 --- a/tests/resources/test_models_resource.py +++ b/tests/resources/test_models_resource.py @@ -1082,3 +1082,181 @@ def test_filter_excludes_all_when_no_match( result = models_resource.get(regions=["ap-southeast-1"]) assert result == [] + + +class TestModelsAddRemoveIncludesCustoms: + """Regression tests for the PR #1916 fix: add()/remove() now operate on + the full project model list (public + custom), not public only. The old + behavior silently wiped any custom model out of Project.Models on every + add/remove because the PATCH payload omitted them.""" + + @pytest.fixture + def mock_client(self): + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.get_cast = Mock() + client.patch_cast = Mock() + return client + + @pytest.fixture + def models_resource(self, mock_client): + return Models(mock_client) + + def test_add_includes_customs_in_patch_payload(self, models_resource): + """add() must preserve already-attached custom models in the PATCH body.""" + public_m = PublicModel(id="pub-1", key="pub-1", name="Pub1", description="") + custom_m = CustomModel( + id="cust-1", + key="cust-1", + name="Cust1", + description="", + max_tokens=2048, + api_url="https://x.example.com", + disabled=False, + ) + models_resource.get = Mock(return_value=[public_m, custom_m]) + models_resource._patch.return_value = {"id": "proj-456"} + + result = models_resource.add("new-id") + + assert result is True + models_resource.get.assert_called_once() + # Critical: the get call should NOT filter by type="public" anymore. + assert models_resource.get.call_args.kwargs.get("type") is None + call_body = models_resource._patch.call_args.kwargs["body"] + assert call_body == {"models": ["pub-1", "cust-1", "new-id"]} + + def test_remove_preserves_unrelated_customs(self, models_resource): + """remove() of a public ID must leave attached customs untouched.""" + public_m = PublicModel(id="pub-1", key="pub-1", name="Pub1", description="") + custom_m = CustomModel( + id="cust-1", + key="cust-1", + name="Cust1", + description="", + max_tokens=2048, + api_url="https://x.example.com", + disabled=False, + ) + models_resource.get = Mock(return_value=[public_m, custom_m]) + models_resource._patch.return_value = {"id": "proj-456"} + + models_resource.remove("pub-1") + + # Custom model must survive. + call_body = models_resource._patch.call_args.kwargs["body"] + assert call_body == {"models": ["cust-1"]} + assert models_resource.get.call_args.kwargs.get("type") is None + + +class TestModelsUpdateCustom: + """Test Models.update_custom() method.""" + + @pytest.fixture + def mock_client(self): + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.patch_cast = Mock() + return client + + @pytest.fixture + def models_resource(self, mock_client): + return Models(mock_client) + + def test_update_custom_api_url_only(self, models_resource): + """update_custom() sends only api_url in body when that's all that's provided.""" + models_resource._patch.return_value = {"data": {"id": "model-1"}} + + result = models_resource.update_custom("model-1", api_url="https://new.example.com/v1") + + assert result is True + models_resource._patch.assert_called_once_with( + "/organizations/org-123/projects/proj-456/custom-models/model-1", + body={"api_url": "https://new.example.com/v1"}, + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_update_custom_all_three_fields(self, models_resource): + """update_custom() sends all provided fields together.""" + models_resource._patch.return_value = {"data": {"id": "model-1"}} + + result = models_resource.update_custom( + "model-1", + api_url="https://x.io", + api_key="sk-new", + max_tokens=1024, + ) + + assert result is True + body = models_resource._patch.call_args.kwargs["body"] + assert body == { + "api_url": "https://x.io", + "api_key": "sk-new", + "max_tokens": 1024, + } + + def test_update_custom_max_tokens_only(self, models_resource): + """update_custom() supports max_tokens-only updates.""" + models_resource._patch.return_value = {"data": {"id": "model-1"}} + + result = models_resource.update_custom("model-1", max_tokens=8192) + + assert result is True + body = models_resource._patch.call_args.kwargs["body"] + assert body == {"max_tokens": 8192} + + def test_update_custom_returns_false_on_error_envelope(self, models_resource): + """update_custom() returns False when response has no data field.""" + models_resource._patch.return_value = {"code": "NOT_FOUND", "message": "missing"} + + result = models_resource.update_custom("model-1", api_url="https://x.io") + + assert result is False + + def test_update_custom_returns_false_when_response_not_dict(self, models_resource): + """update_custom() returns False when response isn't a dict.""" + models_resource._patch.return_value = httpx.Response(404) + + result = models_resource.update_custom("model-1", api_url="https://x.io") + + assert result is False + + +class TestModelsDeleteCustom: + """Test Models.delete_custom() method.""" + + @pytest.fixture + def mock_client(self): + client = Mock() + client.organization_id = "org-123" + client.project_id = "proj-456" + client.delete_cast = Mock() + return client + + @pytest.fixture + def models_resource(self, mock_client): + return Models(mock_client) + + def test_delete_custom_happy_path(self, models_resource): + """delete_custom() hits the right URL and returns True on success.""" + models_resource._delete.return_value = {"data": {"id": "model-1"}} + + result = models_resource.delete_custom("model-1") + + assert result is True + models_resource._delete.assert_called_once_with( + "/organizations/org-123/projects/proj-456/custom-models/model-1", + timeout=DEFAULT_TIMEOUT, + cast_to=dict, + ) + + def test_delete_custom_returns_false_on_error(self, models_resource): + """delete_custom() returns False when response has no data field.""" + models_resource._delete.return_value = {"code": "NOT_FOUND"} + + result = models_resource.delete_custom("model-1") + + assert result is False diff --git a/tests/test_models_custom_live.py b/tests/test_models_custom_live.py new file mode 100644 index 00000000..30702c21 --- /dev/null +++ b/tests/test_models_custom_live.py @@ -0,0 +1,72 @@ +"""Live end-to-end test for the custom-model lifecycle. + +Exercises the customer's exact workflow against a real LayerLens API: +create_custom → update_custom (repoint api_url) → delete_custom → verify gone. + +Skipped unless ``LAYERLENS_STRATIX_API_KEY`` is set. Run with:: + + pytest tests/test_models_custom_live.py -m live +""" + +from __future__ import annotations + +import os +import time +import uuid + +import pytest + +from layerlens import Stratix + + +@pytest.mark.live +def test_custom_model_lifecycle_live() -> None: + if not os.environ.get("LAYERLENS_STRATIX_API_KEY"): + pytest.skip("LAYERLENS_STRATIX_API_KEY not set") + + client = Stratix() + + # Use a unique key per run so the test can re-run cleanly. + suffix = uuid.uuid4().hex[:8] + name = f"sdk-live-custom-{suffix}" + key = f"sdk-live/custom-{suffix}" + + created = client.models.create_custom( + name=name, + key=key, + description="ephemeral live-test custom model", + api_url="https://tunnel-1.example.com/v1", + api_key="sk-live-test", + max_tokens=2048, + ) + assert created is not None, "create_custom returned None" + model_id = created.model_id + assert model_id + + try: + # Repointing api_url is the customer's primary workflow (cloudflared + # tunnels whose URL changes between sessions). + updated = client.models.update_custom( + model_id, + api_url="https://tunnel-2.example.com/v1", + ) + assert updated, "update_custom returned False" + + # Allow a brief moment for the backend to persist (S3 yaml regen + + # Mongo write) — defensive, not strictly required. + time.sleep(0.5) + + # Tear it down completely. + deleted = client.models.delete_custom(model_id) + assert deleted, "delete_custom returned False" + + remaining = client.models.get(type="custom") or [] + assert all(m.id != model_id for m in remaining), f"deleted custom model {model_id} still visible in models.get" + except Exception: + # Best-effort teardown on any assertion / API failure mid-test so a + # broken run doesn't leak project-scoped resources. + try: + client.models.delete_custom(model_id) + except Exception: # noqa: BLE001 + pass + raise diff --git a/tests/test_samples_e2e.py b/tests/test_samples_e2e.py index 3ba00afa..c9a300bf 100644 --- a/tests/test_samples_e2e.py +++ b/tests/test_samples_e2e.py @@ -364,6 +364,8 @@ def mock_stratix(): client.models.add.return_value = True client.models.remove.return_value = True client.models.create_custom.return_value = MagicMock(model_id="model-custom-001") + client.models.update_custom.return_value = True + client.models.delete_custom.return_value = True # --- benchmarks --- benchmark = MagicMock() @@ -572,6 +574,8 @@ def mock_async_stratix(mock_stratix): client.models.add.return_value = True client.models.remove.return_value = True client.models.create_custom.return_value = MagicMock(model_id="model-custom-001") + client.models.update_custom.return_value = True + client.models.delete_custom.return_value = True # --- benchmarks (async) --- benchmark = MagicMock()