From 6b98ff714ff13a7b388bd80e4f4f8d2c04029b89 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Wed, 15 Jul 2026 16:07:37 +0300 Subject: [PATCH] feat: Add Docker Sandbox staging smoke example [INT-976] Manual, resumable Docker Sandbox staging smoke: a headless band-sdk LangGraph agent runs inside a real Docker Sandbox (sbx exec) against staging, proving WS receipt + REST reply, and reconnect across three real interruptions (Wi-Fi cycle, host sleep/wake, sandbox daemon restart). probe.py reuses tests/e2e/baseline's pytest-free toolkit (ResourceManager, UserOps, reply_capture) for dynamic per-run agent/room provisioning + reap, instead of a static staging credential or hand-rolled REST/WS calls. skill/SKILL.md lets an AI client (Codex, Claude Code) drive and resume the workflow across the manual checkpoints; a plain operator runs the same scenario by hand via setup.sh/run.sh/probe.py. Both paths share one state.json and produce the same evidence.md. Deliberately not an automated E2E test: a manual, occasionally-run example under examples/, not tests/e2e/. Live-validated against staging in one continuous run (all 4 checks PASS: initial round trip, Wi-Fi reconnect, sleep/wake recovery, daemon-restart recovery via full re-provision). Two real bugs surfaced only by running live, both worked around: band-sdk's git+https:// install fails inside the sandbox (private SSH-only .claude submodule) so agent.py installs from PyPI instead; the sandbox proxy answers CONNECT with HTTP/1.0, which websockets<=15.0.1 rejects and langgraph-sdk pins, so agent.py overrides to websockets>=16. Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + examples/sandbox/staging-smoke/README.md | 256 ++++++++++++++ examples/sandbox/staging-smoke/agent.py | 155 +++++++++ examples/sandbox/staging-smoke/probe.py | 319 ++++++++++++++++++ examples/sandbox/staging-smoke/run.sh | 70 ++++ examples/sandbox/staging-smoke/setup.sh | 78 +++++ examples/sandbox/staging-smoke/skill/SKILL.md | 177 ++++++++++ .../staging-smoke/skill/agents/openai.yaml | 10 + .../skill/references/requirements.md | 22 ++ .../staging-smoke/skill/scripts/preflight.py | 105 ++++++ .../skill/scripts/record-observation.py | 43 +++ .../skill/scripts/record-phase.py | 47 +++ .../skill/scripts/render-report.py | 127 +++++++ .../staging-smoke/skill/scripts/root.py | 21 ++ examples/sandbox/staging-smoke/state.py | 269 +++++++++++++++ tests/e2e/baseline/fixtures/platform.py | 29 +- tests/e2e/baseline/toolkit/provisioning.py | 17 + tests/e2e/baseline/toolkit/ws.py | 29 +- 18 files changed, 1754 insertions(+), 23 deletions(-) create mode 100644 examples/sandbox/staging-smoke/README.md create mode 100644 examples/sandbox/staging-smoke/agent.py create mode 100644 examples/sandbox/staging-smoke/probe.py create mode 100755 examples/sandbox/staging-smoke/run.sh create mode 100755 examples/sandbox/staging-smoke/setup.sh create mode 100644 examples/sandbox/staging-smoke/skill/SKILL.md create mode 100644 examples/sandbox/staging-smoke/skill/agents/openai.yaml create mode 100644 examples/sandbox/staging-smoke/skill/references/requirements.md create mode 100644 examples/sandbox/staging-smoke/skill/scripts/preflight.py create mode 100644 examples/sandbox/staging-smoke/skill/scripts/record-observation.py create mode 100644 examples/sandbox/staging-smoke/skill/scripts/record-phase.py create mode 100644 examples/sandbox/staging-smoke/skill/scripts/render-report.py create mode 100644 examples/sandbox/staging-smoke/skill/scripts/root.py create mode 100644 examples/sandbox/staging-smoke/state.py diff --git a/.gitignore b/.gitignore index 3f87381bc..9f23a3a33 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ __pycache__/ # E2E baseline run output (scorecards, evidence artifacts) artifacts/ +# Docker Sandbox staging smoke: gitignored per-run state, logs, and evidence.md +examples/sandbox/staging-smoke/.sandbox-smoke/ + # C extensions *.so diff --git a/examples/sandbox/staging-smoke/README.md b/examples/sandbox/staging-smoke/README.md new file mode 100644 index 000000000..4b3110ee0 --- /dev/null +++ b/examples/sandbox/staging-smoke/README.md @@ -0,0 +1,256 @@ +# Docker Sandbox staging smoke + +A reproducible **manual** smoke: a headless `band-sdk` agent runs inside a real +[Docker Sandbox](https://docs.docker.com/ai/sandboxes/) (`sbx exec`) against a +staging Band deployment. It proves WebSocket message receipt, a REST reply, and +— because the operator manually cycles host Wi-Fi while the agent process +keeps running — reconnect behavior behind Docker's proxy. + +This is **not** an automated E2E test. Docker Sandbox policy changes do not +close an already-open WebSocket tunnel, and there is no supported +socket-only fault-injection command, so the network interruption is performed +by a human. The workflow is a two-phase, resumable script: it must not block +through the outage, since turning Wi-Fi off can disconnect the operator too. + +## What this proves + +See `skill/references/requirements.md` for the full requirement-to-evidence +mapping. + +## Prerequisites (one-time) + +```bash +brew install docker/tap/sbx # or see docs.docker.com/ai/sandboxes +sbx login # sign in to Docker (interactive) +sbx policy init balanced # default-deny + common dev/package APIs +uv sync --extra dev # from the repo root +``` + +Every script here except `agent.py` runs inside this repository's own dev +environment, not as a standalone PEP 723 script — they need +`tests/e2e/baseline`, which is dev-only source in this repo, not part of the +published `band-sdk` package (see **How this reuses the SDK's E2E toolkit** +below). `agent.py` is the one exception: it must work as a standalone example +against just the published package, since it runs inside the sandbox as a +customer-style workspace. + +You also need, outside source control: + +- a staging REST URL and WebSocket URL (never production — every script here + rejects the SDK's own production defaults outright) and a staging user/test + key (`BAND_API_KEY_USER`) — this is the *only* Band credential you supply, + since the sandboxed agent's own identity is minted fresh each run (see + below). Add these three to the repo root's `.env.test` — the same file the + rest of the E2E baseline toolkit already uses — rather than a file local to + this directory; +- `SBX_SANDBOX` (a sandbox name) and `SBX_WORKSPACE` (a disposable workspace + path under `$HOME` — **not** an active checkout of this repository, since + the default sandbox mount is read/write and would let the sandbox edit your + working tree live). These are `sbx`-specific, not Band platform config, so + `export` them in your shell rather than adding them to `.env.test`. + +## Two ways to run it + +| Runner | Uses | When to choose it | +|---|---|---| +| Operator | This README plus `setup.sh`, `run.sh`, and `probe.py` | Reproduce or debug the smoke without an AI client. | +| Codex / Claude Code | `skill/SKILL.md`, which orchestrates those same files | Guided progress updates, resume support, and report generation. | + +Both paths write the same `.sandbox-smoke/` state and `evidence.md`; neither is +a second implementation of the scenario. + +To use the skill from an AI client, symlink it into the client's discovery +location (a one-time, idempotent step — `preflight.py` reports a missing or +stale link): + +| Client | Discovery symlink | +|---|---| +| Codex | `$CODEX_HOME/skills/band-sandbox-staging-smoke` → `skill/` | +| Claude Code | `.claude/skills/band-sandbox-staging-smoke` (or `~/.claude/skills/...`) → `skill/` | +| Any other client | Follow this README and run the scripts directly. | + +## Operator flow + +### 1. Preflight and sandbox creation + +```bash +export SBX_SANDBOX=my-sandbox-name +export SBX_WORKSPACE=$HOME/disposable-workspace +./setup.sh +``` + +Runs `skill/scripts/preflight.py` first (the one place every safety check — +`sbx` installed, staging endpoints set and non-production, workspace genuinely +outside this checkout — lives), then creates (or reuses) a disposable +sandbox, allowlists the staging host plus the package hosts needed to +bootstrap (scoped to this sandbox only), copies `agent.py` into the sandbox +workspace, and warms its `uv`-managed environment. It does **not** touch the +Band platform — no agent/room is created yet. + +### 2. Start the headless agent + +```bash +./run.sh +``` + +Keep this terminal open. `run.sh` provisions a fresh Band agent + room +(`probe.py --label provision` — see below), then runs the agent with +`sbx exec`, logging connection, disconnect, and reconnect activity to +`.sandbox-smoke/agent.log`. Wait for its readiness line. + +### 3. Prove the initial round trip + +In a second terminal: + +```bash +uv run probe.py --label initial +``` + +Sends a unique marker as the staging user (mentioning the agent) and waits for +`sandbox-ack:` — the same reply barrier (`wait_for_reply`) every +baseline E2E test in this repo uses. Exits non-zero on timeout or a wrong +reply. + +### 4. Manually interrupt network and verify reconnect + +While `run.sh` keeps running: + +1. turn the host Wi-Fi off; +2. wait until `.sandbox-smoke/agent.log` shows the WebSocket disconnect; +3. turn Wi-Fi back on; +4. wait until the log shows reconnect/rejoin; then + + ```bash + uv run probe.py --label after-wifi-reconnect + ``` + +Success requires the second marker reply from the *same* running agent +process — real, manual, real-network validation of the SDK's reconnect +behavior behind Docker's proxy. `probe.py` can be rerun after a transient +failure (e.g. a timeout while the agent was still warming up) — the evidence +report reflects the *latest* attempt per step, not every historical one. + +### 5. Sleep/wake and daemon restart (mandatory) + +The full run has no optional checks — a report missing either probe renders +INCOMPLETE. + +**Sleep/wake** (observed live: VM and agent both survive; the SDK reconnects +on wake): put the host to sleep for ~1 minute, wake it, confirm the agent log +shows a reconnect, then: + +```bash +uv run probe.py --label after-sleep-wake +uv run skill/scripts/record-observation.py sleep_wake \ + "" +``` + +**Daemon restart** (observed live: the sandbox VM stops and the agent dies +with the daemon — its `sbx exec` attach is severed and the never-persisted +credentials die with the process — so recovery is a full re-provision). +Recording the `daemon-restart` phase first is what authorizes the re-run of +`run.sh` to reuse this run (reaping the dead agent/room itself) instead of +rotating to a fresh one: + +```bash +uv run skill/scripts/record-phase.py daemon-restart +sbx daemon stop +sbx daemon start -d # -d: detached, not foreground +./run.sh # recovery: reaps the dead agent, re-provisions, relaunches +uv run probe.py --label after-daemon-restart # once the readiness line appears +uv run skill/scripts/record-observation.py daemon_restart \ + "" +``` + +The observations are the *behavior* (survived? auto-resumed? what recovery +took) — the report prints them alongside the probe verdicts, and renders +"(not recorded)" visibly when one is missing. + +### 6. Diagnostics and cleanup + +On failure, keep only redacted `.sandbox-smoke/agent.log` excerpts and +`sbx policy log` output — never keys, headers, or config/env dumps. + +```bash +uv run probe.py --label cleanup # deletes the provisioned Band room + agent, ends the run +uv run skill/scripts/render-report.py # writes .sandbox-smoke/evidence.md +``` + +Then stop the agent (`Ctrl-C` in `run.sh`'s terminal), remove the disposable +sandbox, and delete its workspace. A finished (or abandoned) run's state is +archived automatically the next time a run starts — a new smoke can never +silently inherit a previous run's results. + +## How this reuses the SDK's E2E toolkit + +`tests/e2e/baseline/README.md` states its `toolkit/` modules are "pytest-free +and reusable anywhere." `probe.py` (this directory) builds on them directly +instead of hand-rolling REST/WebSocket calls or requiring a static +pre-provisioned staging agent: + +- **`ResourceManager.provision_agent`** mints the sandboxed agent's identity + fresh each run (its own id + api_key) and reaps it on cleanup (including a + failed provisioning attempt, which reaps immediately rather than leaking a + live agent) — there is no standing staging agent to source, rotate, or leak. +- **`UserOps.send_message`** sends the mention-required probe message. +- **`reply_capture` / `wait_for_reply`** observe the room over a real + WebSocket connection and barrier on the agent's reply — the exact + distinction between "turn processed" and "reply frame actually captured" + that every baseline test relies on. + +`probe.py` runs from a full clone of this repository (it imports +`tests.e2e.baseline.*`; `state.repo_root()` finds the repo root, fixed by this +example's own location in the tree), unlike `agent.py`, which must still work +as a standalone example against just the published `band-sdk` package, since +it runs inside the sandbox as a customer-style workspace. + +## Evidence report + +`.sandbox-smoke/` is gitignored and holds durable, redacted evidence: + +- `state.json` — run id, phase, timestamps, sandbox name, version, and + probe/residual-check results; never credentials. +- `agent.log` — the sandboxed agent's own log (connect/disconnect/reconnect). +- `evidence.md` — the rendered PASS/FAIL/INCOMPLETE report, ready to attach to + the work item or copy to your team's evidence store. + +## Design notes + +- Verified locally against the installed `sbx` v0.34.0: `sbx exec` starts a + stopped sandbox and runs commands headlessly, and takes real `-e KEY=value` + and `--workdir` flags (mirroring `docker exec`) — no need to shell out to a + bare `env` command. `sbx create shell PATH` is the right form for a plain + Python workload (sbx's other `create` subcommands are specific coding-agent + environments). `sbx policy allow network --sandbox NAME "host1,host2"` + scopes the allowlist to one sandbox in a single call. `sbx ls --json` + (`ls` is the documented subcommand; `list` also works but is an undocumented + alias) gives a structured way to check whether a sandbox already exists, + instead of pattern-matching table output. +- Also verified: `sbx policy` changes do not tear down an already-open + WebSocket tunnel (confirmed by direct experiment), so the network + interruption in step 4 has to be a real, manual Wi-Fi cycle. +- Discovered via a real sandbox run: installing `band-sdk` from this repo's + own `git+https://...` source (the pattern most other examples in this repo + use) fails inside the sandbox — `uv`/`pip`'s git install does a full clone, + and this repo carries a private, SSH-only `.claude` submodule the sandbox + has no credentials for, so the clone's submodule step fails even though + `.claude` has nothing to do with the `band` package. `agent.py` and this + script install `band-sdk[langgraph]` from PyPI instead, which needs no + clone at all. +- Discovered via a real sandbox run (the smoke's first genuinely valuable + catch): the sandbox proxy answers `CONNECT` with `HTTP/1.0 200`, which + `websockets <= 15.0.1` rejects outright (`InvalidProxyMessage: did not + receive a valid HTTP response from proxy`; fixed in websockets 16.0, and no + fixed 15.x exists) — while REST keeps working, since `httpx` tolerates + HTTP/1.0. `langgraph-sdk` pins `websockets<16`, forcing the broken version + onto every `band-sdk[langgraph]` install. So *any* langgraph-based band + agent behind an HTTP/1.0-answering proxy (Docker Sandbox's included, and + many corporate proxies) can never open its WebSocket. `agent.py` carries a + `[tool.uv] override-dependencies = ["websockets>=16"]` as the workaround — + safe because the deterministic graph never uses langgraph-sdk's own + WebSocket client, which is all that pin protects. Worth an upstream fix: + `phoenix-channels-python-client` declares `websockets>=10.0`, far below + what proxy support actually requires. +- Sandbox CLI behavior changes between releases — re-check `sbx --version` + and `sbx --help` before relying on any of the above after + upgrading. diff --git a/examples/sandbox/staging-smoke/agent.py b/examples/sandbox/staging-smoke/agent.py new file mode 100644 index 000000000..daa48a62a --- /dev/null +++ b/examples/sandbox/staging-smoke/agent.py @@ -0,0 +1,155 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = ["band-sdk[langgraph]"] +# +# [tool.uv] +# override-dependencies = ["websockets>=16"] +# /// +# The override matters behind a proxy (Docker Sandbox included): the proxy +# answers CONNECT with `HTTP/1.0 200`, which websockets <= 15.0.1 rejects +# ("did not receive a valid HTTP response from proxy" — fixed in 16.0), so +# the SDK's WebSocket can never connect while REST keeps working. langgraph-sdk +# pins websockets<16 (no fixed 15.x exists), forcing the broken version unless +# overridden. Safe here: this graph runs locally and never uses langgraph-sdk's +# own WebSocket client, which is the only thing that pin protects. +""" +Deterministic sandboxed agent for the Docker Sandbox staging smoke. + +Installs `band-sdk[langgraph]` from PyPI, not this repo's own git source +(unlike most examples elsewhere in this repo): a `uv`/`pip` git install does a +full clone, and this repo carries a private, SSH-only `.claude` submodule +(`.gitmodules`) that a sandboxed environment has no credentials for — its +submodule init fails there even though it has nothing to do with the `band` +package itself. PyPI's `band-sdk` is a plain sdist/wheel with no such step, +and (checked when this was written) it isn't behind git HEAD. + +Runs headlessly inside a real Docker Sandbox (`sbx exec`) against staging. It +calls no model: on a mentioned message it extracts a `marker:` from the +content, resolves the sending user's mention handle via `band_get_participants`, +and replies with `sandbox-ack:` via `band_send_message` — proving the +SDK's normal WebSocket-receive + REST-reply round trip from inside the sandbox. + +`LangGraphAdapter` never relays a graph's plain text back to the room +automatically, so the reply must go through `band_send_message` explicitly; +calling it also satisfies the platform's mention requirement. + +Unlike other examples, this one's own Band identity (`BAND_AGENT_ID` / +`BAND_API_KEY`) is not read from a checked-in `agent_config.yaml` via +`load_agent_config`. `run.sh` provisions the agent fresh for each run (see +`probe.py --label provision`, which reuses the SDK's own E2E baseline +toolkit) and injects the newly-minted credentials as environment variables; +there is no static credential to source here. + +Run with (invoked by run.sh inside the sandbox): + uv run agent.py +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import re +from typing import Any + +from langgraph.graph import END, MessagesState, StateGraph + +from band import Agent +from band.adapters import LangGraphAdapter +from band.runtime.types import normalize_handle + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s" +) +logger = logging.getLogger(__name__) + +_MARKER_PATTERN = re.compile(r"marker:(\S+)") + + +def extract_marker(content: str) -> str: + match = _MARKER_PATTERN.search(content) + if not match: + raise ValueError(f"no marker found in message content: {content!r}") + return match.group(1) + + +def ensure_not_error(result: Any, *, tool_name: str) -> Any: + """Band platform tools return an *error string* instead of raising on an + unexpected failure (see `AgentTools.execute_tool_call`'s docstring) — that + convention exists for LLM callers, which read the string and react. This + graph has no LLM to notice it, so check explicitly and raise a diagnosable + error instead of, say, iterating an error string as if it were the + expected list of participants. + """ + if isinstance(result, str): + raise RuntimeError(f"{tool_name} failed: {result}") + return result + + +def deterministic_graph_factory(band_tools: list[Any]) -> Any: + """Build a no-LLM graph: read the marker, mention the sender, ack it.""" + tools_by_name = {tool.name: tool for tool in band_tools} + get_participants = tools_by_name["band_get_participants"] + send_message = tools_by_name["band_send_message"] + + async def respond(state: MessagesState) -> dict[str, list[Any]]: + marker = extract_marker(str(state["messages"][-1].content)) + + participants = ensure_not_error( + await get_participants.ainvoke({}), tool_name="band_get_participants" + ) + sender = next( + (p for p in participants if p.get("type") != "Agent" and p.get("handle")), + None, + ) + if sender is None: + raise RuntimeError("no mentionable (non-agent) participant in room") + handle = normalize_handle(sender["handle"]) + if handle is None: + raise RuntimeError(f"participant has no handle to mention: {sender!r}") + + ensure_not_error( + await send_message.ainvoke( + {"content": f"sandbox-ack:{marker}", "mentions": [handle]} + ), + tool_name="band_send_message", + ) + return {"messages": []} + + graph = StateGraph(MessagesState) + graph.add_node("respond", respond) + graph.set_entry_point("respond") + graph.add_edge("respond", END) + return graph.compile() + + +async def main() -> None: + ws_url = os.getenv("BAND_WS_URL") + rest_url = os.getenv("BAND_REST_URL") + agent_id = os.getenv("BAND_AGENT_ID") + api_key = os.getenv("BAND_API_KEY") + + if not ws_url: + raise ValueError("BAND_WS_URL environment variable is required") + if not rest_url: + raise ValueError("BAND_REST_URL environment variable is required") + if not agent_id: + raise ValueError("BAND_AGENT_ID environment variable is required") + if not api_key: + raise ValueError("BAND_API_KEY environment variable is required") + + adapter = LangGraphAdapter(graph_factory=deterministic_graph_factory) + agent = Agent.create( + adapter=adapter, + agent_id=agent_id, + api_key=api_key, + ws_url=ws_url, + rest_url=rest_url, + ) + + logger.info("Sandbox smoke agent starting (agent_id=%s)", agent_id) + await agent.run() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/sandbox/staging-smoke/probe.py b/examples/sandbox/staging-smoke/probe.py new file mode 100644 index 000000000..be0d781d7 --- /dev/null +++ b/examples/sandbox/staging-smoke/probe.py @@ -0,0 +1,319 @@ +""" +Band-platform driver for the Docker Sandbox staging smoke. + +Unlike `agent.py` (which must run as a standalone example against just the +published `band-sdk` package inside the sandbox), this script runs on the +operator's host from within this repository's own dev environment +(`uv sync --extra dev`) and reuses `tests/e2e/baseline`'s pytest-free toolkit +directly — `ResourceManager` (dynamic agent/room provisioning + reap + +orphan sweep), `UserOps.send_message`, and `reply_capture`/`wait_for_reply` — +instead of hand-rolling REST/WS calls or requiring a static pre-provisioned +staging agent. It has no PEP 723 header (unlike every sibling script in this +directory except `agent.py`) because `tests/e2e/baseline` is dev-only source +in this repo, not part of the published package a PEP 723 isolated venv could +install; it needs the surrounding project's own synced environment. It is an +internal repo tool, not a redistributable customer example, even though it +lives under `examples/` for discovery alongside the runbook and skill. + +One entry point, six labels, because they all share the same toolkit +construction (settings, REST client): + + uv run probe.py --label provision # run.sh: mint agent+room + uv run probe.py --label initial # first round trip + uv run probe.py --label after-wifi-reconnect # after the Wi-Fi cycle + uv run probe.py --label after-sleep-wake # after host sleep/wake + uv run probe.py --label after-daemon-restart # after the daemon bounce + uv run probe.py --label cleanup # final teardown + +`provision` prints exactly four `KEY=value` lines to stdout — the fresh +agent's `BAND_AGENT_ID`/`BAND_API_KEY` plus the *validated* `BAND_WS_URL`/ +`BAND_REST_URL` (the same values `load_settings()` guarded, alias-resolved +from `.env.test`) — for `run.sh` to capture and inject into the sandboxed +process's environment. Nothing else goes to stdout, and no credential is +written to `.sandbox-smoke/state.json` (see `state.py`). Emitting the URLs +here keeps run.sh's agent on exactly the endpoints the production guard +checked, instead of whatever the raw shell environment happens to hold. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +import uuid + +import state + +# Reuse tests/e2e/baseline's toolkit directly (see module docstring) — insert +# the repo root (state.repo_root() is fixed by this example's own location in +# the tree, not a hardcoded parents[N] hop count re-derived here) so +# `tests.e2e.baseline.*` imports the same way `conftest.py` does for pytest. +sys.path.insert(0, str(state.repo_root())) + +from band_rest import AsyncRestClient # noqa: E402 + +from tests.e2e.baseline.settings import BandEndpoints, BaselineSettings # noqa: E402 +from tests.e2e.baseline.toolkit.capture import reply_capture # noqa: E402 +from tests.e2e.baseline.toolkit.provisioning import ( # noqa: E402 + ResourceManager, + user_rest_client, +) +from tests.e2e.baseline.toolkit.user_ops import UserOps # noqa: E402 +from tests.e2e.baseline.toolkit.ws import user_ws_observer # noqa: E402 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + stream=sys.stderr, # `provision` uses stdout as its KEY=value protocol +) +logger = logging.getLogger(__name__) + + +def load_settings() -> BaselineSettings: + """Load Band config, rejecting the SDK's own production defaults outright. + + Reads the *declared* defaults straight off `BandEndpoints`'s field + metadata rather than comparing against a second, hand-typed copy of the + URLs (which could silently drift from the real default) — and rather + than constructing a fresh `BandEndpoints()` (which would just read the + same environment `settings.endpoints` did, making the comparison + vacuous). + """ + settings = BaselineSettings() + production_rest_url = BandEndpoints.model_fields["rest_url"].default + production_ws_url = BandEndpoints.model_fields["ws_url"].default + if settings.endpoints.rest_url == production_rest_url: + raise ValueError( + "BAND_REST_URL is unset or points at production; set it to the " + "staging REST URL before running this smoke." + ) + if settings.endpoints.ws_url == production_ws_url: + raise ValueError( + "BAND_WS_URL is unset or points at production; set it to the " + "staging WebSocket URL before running this smoke." + ) + if not settings.credentials.api_key_user: + raise ValueError("BAND_API_KEY_USER is required (the staging user key)") + return settings + + +def bootstrap() -> tuple[BaselineSettings, AsyncRestClient]: + """Settings + a user-authenticated REST client — the one piece every + label below needs, built once instead of three separate times.""" + settings = load_settings() + return settings, user_rest_client(settings) + + +async def _reap_recorded_resources( + manager: ResourceManager, run_state: state.SmokeState +) -> bool: + """Reap the room/agent ids recorded in ``run_state``; True if all reaped. + + Each resource is reaped independently: a failed room delete must never + leave the (live, credentialed) agent running because its reap was skipped. + An id is cleared from the state only after *its* reap succeeds — a failed + reap keeps the id recorded, so the resource stays retryable instead of + surviving only as a log line (rooms especially: the orphan sweep covers + aged agents by name prefix, but nothing ever sweeps rooms). Used both when + re-provisioning over a dead process (daemon-restart recovery, a crashed + launch) and by terminal cleanup. + """ + all_reaped = True + for id_field, reap, kind in ( + ("room_id", manager.reap_room, "room"), + ("agent_id", manager.reap_agent, "agent"), + ): + resource_id = getattr(run_state, id_field) + if not resource_id: + continue + try: + await reap(resource_id) + except Exception: + all_reaped = False + logger.exception( + "Failed to reap %s %s — id kept in state for a retry", + kind, + resource_id, + ) + continue + logger.info("Reaped %s %s", kind, resource_id) + setattr(run_state, id_field, "") + if not run_state.agent_id: + run_state.agent_name = "" + return all_reaped + + +async def provision(sandbox_name: str, sbx_version: str, sdk_version: str) -> None: + settings, client = bootstrap() + run_state = state.begin_provision() + manager = ResourceManager( + user_client=client, settings=settings, run_id=run_state.run_id + ) + + # Reap leftovers this workflow itself may have abandoned (a crashed prior + # run's agent is invisible to any later run once its state file rotates) — + # the toolkit's sweep is prefix- and age-guarded, so nothing else's + # resources are ever touched. + await manager.sweep_orphans() + # Re-provisioning within a run (daemon-restart recovery, a crashed + # launch): the recorded agent/room belong to a dead process — replace + # them. Refuse to stack new resources on top of ones that failed to reap + # (their ids stay recorded, so a rerun retries the reap first). + if not await _reap_recorded_resources(manager, run_state): + state.save(run_state) + logger.error( + "Could not reap this run's previous resources; rerun to retry " + "before provisioning replacements." + ) + raise SystemExit(1) + + # Each id is persisted to state.json the moment its resource exists, + # BEFORE the next fallible step — so any failure (or even a SIGKILL) + # leaves a retryable record on disk, never a resource whose id survives + # only in a log line. The participant-add is deliberately a separate step + # after the room id is recorded, not provision_room's participants= + # argument, which would create the room and then raise past us on a + # failed add with the room id still unrecorded. + run_state.sandbox_name = sandbox_name + run_state.sbx_version = sbx_version + run_state.sdk_version = sdk_version + try: + agent = await manager.provision_agent("sandbox") + run_state.agent_id = agent.id + run_state.agent_name = agent.name + state.save(run_state) + # A descriptive title, not the platform's default untitled-room + # label, so an operator watching the Band UI can find this run's + # room among others at a glance. + room_id = await manager.provision_room( + title=f"Sandbox smoke — {run_state.run_id}" + ) + run_state.room_id = room_id + state.save(run_state) + await UserOps(client).add_participant(room_id, agent.id) + except Exception: + logger.exception( + "Provisioning failed partway; reaping what was created so far " + "so nothing is left running on staging" + ) + if not await _reap_recorded_resources(manager, run_state): + logger.error( + "Some resources could not be reaped — their ids remain in " + "state.json; rerun `probe.py --label cleanup` to retry." + ) + state.save(run_state) + raise + + logger.info("Provisioned agent %s and room %s", agent.id, room_id) + # The only stdout output: run.sh captures these four lines verbatim. + # The URLs are the validated settings values, so the sandboxed agent can + # only ever target the endpoints the production guard checked. + sys.stdout.write(f"BAND_AGENT_ID={agent.id}\n") + sys.stdout.write(f"BAND_API_KEY={agent.api_key}\n") + sys.stdout.write(f"BAND_WS_URL={settings.endpoints.ws_url}\n") + sys.stdout.write(f"BAND_REST_URL={settings.endpoints.rest_url}\n") + + +async def _run_probe(label: str) -> None: + settings, client = bootstrap() + run_state = state.load() + user_ops = UserOps(client) + marker = uuid.uuid4().hex[:12] + + async with ( + user_ws_observer(settings) as tracking, + reply_capture( + tracking, + run_state.room_id, + user_ops=user_ops, + settings=settings, + deadline_s=settings.e2e_timeout, + ) as capture, + ): + logger.info("Sending marker %s (label=%s)", marker, label) + mid = await user_ops.send_message( + run_state.room_id, + f"marker:{marker}", + mention_id=run_state.agent_id, + mention_name=run_state.agent_name, + ) + try: + replies = await capture.wait_for_reply(mid, run_state.agent_id) + replies.assert_contains_any([f"sandbox-ack:{marker}"]) + except (TimeoutError, AssertionError) as error: + run_state.probes.append( + state.ProbeResult( + label=label, marker=marker, passed=False, detail=str(error) + ) + ) + state.save(run_state) + logger.error("Probe %s failed: %s", label, error) + raise SystemExit(1) from error + + run_state.probes.append(state.ProbeResult(label=label, marker=marker, passed=True)) + state.save(run_state) + logger.info("Probe %s passed (marker=%s)", label, marker) + + +async def cleanup() -> None: + """Terminal teardown: reap the run's room + agent and end the run. + + This is the one place `probe.py` writes `phase` — the run's shared + terminal transition, so the plain operator workflow (which never calls + `record-phase.py`) still produces a finished run that the next + `provision` rotates instead of resuming. Mid-run resource replacement + (daemon-restart recovery) is handled by `provision` itself, never by + calling this. + """ + settings, client = bootstrap() + run_state = state.load() + manager = ResourceManager( + user_client=client, settings=settings, run_id=run_state.run_id + ) + if not await _reap_recorded_resources(manager, run_state): + # Don't end the run over live resources: the surviving ids stay in + # state.json, so rerunning `--label cleanup` retries exactly them. + state.save(run_state) + logger.error("Cleanup incomplete — rerun `probe.py --label cleanup` to retry.") + raise SystemExit(1) + run_state.phase = "completed" + state.save(run_state) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--label", + required=True, + # The three "after-*" labels are the recovery probes, one per + # interruption scenario — each proves the same marker/reply round + # trip after its interruption, rather than an operator eyeballing + # the log. + choices=[ + "provision", + "initial", + "after-wifi-reconnect", + "after-sleep-wake", + "after-daemon-restart", + "cleanup", + ], + ) + parser.add_argument("--sandbox-name", default="") + parser.add_argument("--sbx-version", default="") + parser.add_argument("--sdk-version", default="") + args = parser.parse_args() + + match args.label: + case "provision": + asyncio.run( + provision(args.sandbox_name, args.sbx_version, args.sdk_version) + ) + case "cleanup": + asyncio.run(cleanup()) + case label: + asyncio.run(_run_probe(label)) + + +if __name__ == "__main__": + main() diff --git a/examples/sandbox/staging-smoke/run.sh b/examples/sandbox/staging-smoke/run.sh new file mode 100755 index 000000000..aa3520549 --- /dev/null +++ b/examples/sandbox/staging-smoke/run.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Provisions the Band agent/room fresh (via `probe.py --label provision` — +# never a static credential), then launches the deterministic agent inside +# the sandbox. +# +# Keep this terminal open: it stays in the foreground so its log shows the +# WebSocket connect/disconnect/reconnect activity the smoke is proving. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +: "${SBX_SANDBOX:?Set SBX_SANDBOX to the sandbox created by setup.sh}" +: "${SBX_WORKSPACE:?Set SBX_WORKSPACE to the same disposable workspace setup.sh used}" +# No BAND_* env vars required here: probe.py reads them from the repo root's +# .env.test (see README), validates them against the production guard, and +# hands the validated values back on stdout below — so the sandboxed agent +# can only ever target the endpoints the guard checked, not whatever the raw +# shell environment happens to hold. + +mkdir -p .sandbox-smoke +chmod 700 .sandbox-smoke + +SBX_VERSION="$(sbx version | head -1)" +# Query the exact environment agent.py runs in — the PEP 723 script env that +# setup.sh warmed with `uv sync --script agent.py` (override included) — via +# `uv python find --script`, which resolves without installing anything. Not +# a separate `uv run --with` env, which could resolve a different band-sdk +# than the one actually executed. +SDK_VERSION="$(sbx exec --workdir "$SBX_WORKSPACE" "$SBX_SANDBOX" \ + sh -c '"$(uv python find --script agent.py)" -c "import band; print(band.__version__)"' \ + 2>/dev/null || echo unknown)" + +echo "Provisioning a fresh Band agent + room for this run..." +PROVISION_OUTPUT="$(uv run probe.py --label provision \ + --sandbox-name "$SBX_SANDBOX" --sbx-version "$SBX_VERSION" --sdk-version "$SDK_VERSION")" + +if [[ "$(echo "$PROVISION_OUTPUT" | wc -l | tr -d ' ')" != "4" ]]; then + echo "Provisioning stdout was not exactly the expected four KEY=value lines; aborting." >&2 + exit 1 +fi + +BAND_AGENT_ID="$(echo "$PROVISION_OUTPUT" | sed -n 's/^BAND_AGENT_ID=//p')" +BAND_API_KEY="$(echo "$PROVISION_OUTPUT" | sed -n 's/^BAND_API_KEY=//p')" +BAND_WS_URL="$(echo "$PROVISION_OUTPUT" | sed -n 's/^BAND_WS_URL=//p')" +BAND_REST_URL="$(echo "$PROVISION_OUTPUT" | sed -n 's/^BAND_REST_URL=//p')" + +if [[ -z "$BAND_AGENT_ID" || -z "$BAND_API_KEY" || -z "$BAND_WS_URL" || -z "$BAND_REST_URL" ]]; then + echo "Provisioning did not return the agent credentials and endpoints; aborting." >&2 + exit 1 +fi + +echo "Agent provisioned: $BAND_AGENT_ID" +echo "Starting the sandboxed agent (sbx exec)..." +echo "Log: .sandbox-smoke/agent.log" + +# The freshly-minted credentials are passed via `sbx exec`'s own `-e` flags +# (verified against `sbx exec --help`, sbx v0.34.0 — it mirrors `docker exec`), +# not a file, so nothing sensitive touches disk. This is visible in the +# host's own process listing for as long as `sbx exec` runs — acceptable for +# an operator-controlled local smoke; a later, separate credential-injection +# effort can replace this with Docker's proxy-based injection so the value +# never appears even there. +sbx exec \ + -e "BAND_AGENT_ID=$BAND_AGENT_ID" \ + -e "BAND_API_KEY=$BAND_API_KEY" \ + -e "BAND_WS_URL=$BAND_WS_URL" \ + -e "BAND_REST_URL=$BAND_REST_URL" \ + --workdir "$SBX_WORKSPACE" \ + "$SBX_SANDBOX" uv run agent.py \ + 2>&1 | tee -a .sandbox-smoke/agent.log diff --git a/examples/sandbox/staging-smoke/setup.sh b/examples/sandbox/staging-smoke/setup.sh new file mode 100755 index 000000000..23c2bb8bb --- /dev/null +++ b/examples/sandbox/staging-smoke/setup.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Preflight + disposable sandbox creation for the Docker Sandbox staging smoke. +# +# Does NOT provision the Band agent/room — that happens in run.sh (via +# `probe.py --label provision`), right before the sandboxed process starts, so +# the freshly-minted credentials never touch disk (see probe.py's docstring). +# +# One-time prerequisites (see README.md): `sbx login`, `sbx policy init balanced`, +# `uv sync --extra dev` (this script and probe.py run via this repo's own dev +# environment, not a standalone PEP 723 script — see probe.py's docstring). +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +: "${SBX_SANDBOX:?Set SBX_SANDBOX to the sandbox name to create/use}" +: "${SBX_WORKSPACE:?Set SBX_WORKSPACE to a disposable workspace path under \$HOME (not this checkout)}" + +# All other safety checks (sbx installed, staging endpoints set and +# non-production, workspace genuinely outside this checkout) live in +# preflight.py — the one place that logic exists, so setup.sh doesn't carry a +# second, independently-maintained copy of the same checks. +uv run skill/scripts/preflight.py + +mkdir -p .sandbox-smoke +chmod 700 .sandbox-smoke +mkdir -p "$SBX_WORKSPACE" + +SANDBOX_EXISTS="$(sbx ls --json | python3 -c " +import json, os, sys +# .get(...) or []: a machine with zero sandboxes may report null / omit the key. +sandboxes = json.load(sys.stdin).get('sandboxes') or [] +name = os.environ['SBX_SANDBOX'] +print('yes' if any(s['name'] == name for s in sandboxes) else 'no') +")" + +if [[ "$SANDBOX_EXISTS" == "yes" ]]; then + echo "Sandbox '$SBX_SANDBOX' already exists; reusing it." +else + echo "Creating sandbox '$SBX_SANDBOX' at $SBX_WORKSPACE..." + # `shell`: this smoke runs a plain Python script, not any of sbx's built-in + # coding-agent environments (claude/codex/copilot/...). Verified against + # `sbx create --help` / `sbx create shell --help` (sbx v0.34.0). + sbx create --name "$SBX_SANDBOX" shell "$SBX_WORKSPACE" +fi + +# Allowlist only the staging hosts and the package hosts needed to install the +# SDK-under-test, scoped to this sandbox alone (`--sandbox`, not the global +# policy). Both the REST and the WebSocket hostnames: a deployment may serve +# them from different hosts, and a missing WS allowlist entry would surface +# only later as an opaque probe timeout. One call, not one per host — +# `sbx policy allow network` takes a comma-separated list (verified against +# `sbx policy allow network --help`). +STAGING_HOSTS="$(uv run python -c " +import sys +sys.path.insert(0, '.') +from urllib.parse import urlparse +import probe +settings = probe.load_settings() +hosts = { + urlparse(settings.endpoints.rest_url).hostname, + urlparse(settings.endpoints.ws_url).hostname, +} +print(','.join(sorted(h for h in hosts if h))) +")" +sbx policy allow network --sandbox "$SBX_SANDBOX" \ + "$STAGING_HOSTS,pypi.org,files.pythonhosted.org,github.com" + +echo "Copying agent.py into the sandbox workspace..." +cp "$(pwd)/agent.py" "$SBX_WORKSPACE/agent.py" + +echo "Warming the sandbox's uv-managed environment for agent.py's dependencies..." +# `uv sync --script` resolves and installs agent.py's own PEP 723 environment — +# including its [tool.uv] override-dependencies (websockets>=16, required for +# the SDK's WebSocket to work through the sandbox proxy; see agent.py) — so +# this warms exactly the env `uv run agent.py` will use, not a lookalike. +sbx exec --workdir "$SBX_WORKSPACE" "$SBX_SANDBOX" uv sync --script agent.py + +echo "Setup complete. Next: ./run.sh" diff --git a/examples/sandbox/staging-smoke/skill/SKILL.md b/examples/sandbox/staging-smoke/skill/SKILL.md new file mode 100644 index 000000000..2d968fe41 --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/SKILL.md @@ -0,0 +1,177 @@ +--- +name: band-sandbox-staging-smoke +description: Run the Docker Sandbox staging smoke for band-sdk-python — a manual, resumable full run proving a headless agent inside a real Docker Sandbox receives a WebSocket message, replies over REST, and recovers from three interruptions (an operator Wi-Fi cycle, host sleep/wake, and a sandbox daemon restart). Use when asked to run, resume, or report on this smoke. +--- + +# Band Sandbox Staging Smoke + +Drives the same files an operator would run by hand (`setup.sh`, `run.sh`, +`probe.py`) plus the scripts in this directory. Neither path is a second +implementation of the scenario — both read and write the same +`.sandbox-smoke/state.json` and produce the same `evidence.md`. + +**This is a manual, occasionally-run example, not an automated test.** Do not +try to make it non-interactive end to end — the Wi-Fi step genuinely requires +a human, and the workflow is designed around that. + +## Safety rules + +- **Never print, log, or write a secret value** — not `BAND_API_KEY_USER`, not + the sandboxed agent's own api_key, not any header. `preflight.py` and + `probe.py` only ever report *presence* and *pass/fail*, never values. + `state.json` never contains a credential field — see `state.py`. +- **Staging only.** `preflight.py` and `probe.py` reject the SDK's own + production defaults outright; if either fails on that check, stop — do not + work around it by hand-supplying a production URL. +- **Stop at the Wi-Fi checkpoint.** After the initial probe passes, end your + active turn instead of trying to stay connected or poll for connectivity — + turning Wi-Fi off can disconnect the operator from you as well as the + sandbox. Resume only when the operator confirms connectivity is back. +- **On failure, keep only redacted diagnostics** — `.sandbox-smoke/agent.log` + and `sbx policy log` output, never raw env dumps or config files. + +## Workflow + +Run every command from `examples/sandbox/staging-smoke/`. + +### 1. Start + +```bash +uv run skill/scripts/record-phase.py started +``` + +State the scope out loud to the operator: this proves a headless SDK agent's +WebSocket receipt + REST reply from inside a real Docker Sandbox, and its +recovery from **three** interruptions — a manual Wi-Fi cycle, a host +sleep/wake (both need the operator), and a sandbox daemon restart (you run +that one). All three are mandatory; a report missing any renders INCOMPLETE. + +### 2. Preflight + +```bash +uv run skill/scripts/preflight.py +``` + +Report each PASS/FAIL line verbatim (it already omits values). Any FAIL stops +the run here — fix the missing prerequisite (see `README.md`) before +continuing. + +### 3. Sandbox creation + +```bash +./setup.sh +``` + +This creates (or reuses) the disposable sandbox and installs the SDK-under-test +inside it. It does **not** touch the Band platform — no agent/room exists yet. +(`setup.sh` also runs `preflight.py` itself as its own first step — this is a +backstop for the plain operator workflow, which never calls step 2 above; it's +not a second, different check.) + +### 4. Start the agent + +Run `./run.sh` **in the background** (it provisions a fresh Band agent + room +via `probe.py --label provision`, then keeps running in the foreground to log +connect/disconnect/reconnect activity — you need to keep driving the +workflow, so launch it detached and tail its log instead of blocking on it). +Wait until `.sandbox-smoke/agent.log` shows the agent's non-secret readiness +line before continuing, and tell the operator the agent is up. + +### 5. Initial probe + +```bash +uv run probe.py --label initial +``` + +A non-zero exit means the round trip failed — report the (redacted) reason +from its output and stop; do not continue to the Wi-Fi step. + +### 6. Wi-Fi checkpoint — end your turn here + +```bash +uv run skill/scripts/record-phase.py awaiting-wifi-recovery +``` + +Print its output (the exact resume instruction) to the operator, then **stop**. +Do not wait, poll, or try to detect reconnection yourself. + +### 7. Resume after Wi-Fi is restored + +When the operator confirms connectivity is back (a new message, not a +continuation of the same blocked turn): + +```bash +uv run skill/scripts/preflight.py # re-check prerequisites +uv run probe.py --label after-wifi-reconnect +``` + +A non-zero exit from the probe means reconnect did not complete — report the +reason; the run is a FAIL, not a retry loop. + +### 8. Sleep/wake checkpoint — end your turn here (mandatory) + +This check is **not optional** — a report missing it renders INCOMPLETE. + +```bash +uv run skill/scripts/record-phase.py awaiting-sleep-wake +``` + +Print its output (sleep the host ~1 minute, wake, resume), then **stop**, the +same as the Wi-Fi checkpoint — the host sleeping suspends you too. + +When the operator confirms they're back: check `sbx ls` (expect the sandbox +still `running`) and the tail of `.sandbox-smoke/agent.log` (expect a +`WebSocket reconnected` line after the wake), then: + +```bash +uv run probe.py --label after-sleep-wake +uv run skill/scripts/record-observation.py sleep_wake \ + "" +``` + +The observation is the *behavior*, not a bare pass/fail — the probe row is +the verdict; this is the explanation the report prints beside it. + +### 9. Daemon restart (mandatory — you run this one) + +No operator handoff needed: restarting the sandbox daemon does not disconnect +you (it only affects sandboxes). **Expect the agent to die** — the daemon stop +severs the `sbx exec` attach and stops the sandbox VM, and the freshly-minted +credentials died with the process (they are deliberately never on disk), so +recovery is a full re-provision. The `daemon-restart` phase recorded first is +what authorizes `provision` to reuse this run (and reap the dead agent/room +itself) instead of rotating to a fresh one: + +```bash +uv run skill/scripts/record-phase.py daemon-restart +sbx daemon stop +sbx daemon start -d # -d: detached, not foreground +sbx ls # expect the sandbox: stopped +./run.sh # recovery: reaps the dead agent, re-provisions, relaunches +# wait for the readiness line in .sandbox-smoke/agent.log, then: +uv run probe.py --label after-daemon-restart +uv run skill/scripts/record-observation.py daemon_restart \ + "" +``` + +### 10. Cleanup and report + +```bash +uv run probe.py --label cleanup # reaps the room + agent, ends the run +uv run skill/scripts/render-report.py +``` + +(If the run must be abandoned as failed instead, record that explicitly: +`uv run skill/scripts/record-phase.py failed` — the report then renders FAIL.) + +Read back `.sandbox-smoke/evidence.md` and give the operator a concise +PASS/FAIL/INCOMPLETE summary plus the report path. Then stop the sandboxed +`run.sh` process, remove the disposable sandbox, and delete its workspace +(see `README.md`'s cleanup section) — `probe.py --label cleanup` only reaps +the Band-side room/agent, not the sandbox itself. + +## Reference + +`references/requirements.md` maps each requirement to the evidence this +smoke produces and the script that produces it — read it if asked to justify +what a PASS actually proves. diff --git a/examples/sandbox/staging-smoke/skill/agents/openai.yaml b/examples/sandbox/staging-smoke/skill/agents/openai.yaml new file mode 100644 index 000000000..6514d89b3 --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/agents/openai.yaml @@ -0,0 +1,10 @@ +# Optional Codex-specific display metadata only. Must not contain +# instructions, commands, credentials, or any behavior needed to run the +# smoke — that all lives in ../SKILL.md, which every client (Codex, Claude +# Code, or a plain operator) reads and runs the same way. Other clients +# ignore this file entirely. +name: band-sandbox-staging-smoke +description: >- + Manual, resumable Docker Sandbox staging smoke for band-sdk-python: + headless agent round trip, operator-driven Wi-Fi reconnect proof, evidence + report. diff --git a/examples/sandbox/staging-smoke/skill/references/requirements.md b/examples/sandbox/staging-smoke/skill/references/requirements.md new file mode 100644 index 000000000..6c6e265f5 --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/references/requirements.md @@ -0,0 +1,22 @@ +# Requirements and evidence mapping + +Reference for the skill workflow (`SKILL.md`) and for anyone auditing a run's +`evidence.md`. Maps what this smoke needs to prove, one-to-one with what it +actually produces as evidence. + +| Requirement | Evidence | Produced by | +|---|---|---| +| Headless SDK agent in a real sandbox | `run.sh` starts `agent.py` with `sbx exec` inside a real Docker Sandbox | `run.sh` | +| Staging setup, not production | Staging REST/WS URLs required; the SDK's own production defaults are rejected outright | `probe.py`'s `load_settings()` | +| Dynamic, reaped agent identity | The sandboxed agent is registered fresh each run (own id + key) via `ResourceManager.provision_agent`, never a static pre-provisioned credential; a failed provisioning attempt reaps the agent immediately rather than leaking it | `probe.py --label provision` | +| WebSocket receipt | A scripted `marker:` message reaches the sandboxed agent over its real WebSocket connection | `probe.py --label initial` / `agent.py` | +| REST reply | The agent replies `sandbox-ack:` via `band_send_message`; the probe observes it via `reply_capture`'s `wait_for_reply` | `agent.py` / `probe.py` | +| Reconnect after network loss | The operator manually disables/enables Wi-Fi while the agent process keeps running; a second marker/reply round trip proves the same process reconnected. Retrying after a transient failure is safe — the report reflects the latest attempt per step | `probe.py --label after-wifi-reconnect` | +| Recovery after host sleep/wake (mandatory) | The operator suspends and wakes the host; a marker/reply round trip proves recovery (observed live: VM + agent survive, SDK reconnects on wake) | `probe.py --label after-sleep-wake` + a behavior note in `residual_checks` | +| Recovery after daemon restart (mandatory) | `sbx daemon stop`/`start -d` kills the sandbox VM and agent (observed live); the documented recovery path (reap + re-provision + relaunch) is then proven with a marker/reply round trip | `probe.py --label after-daemon-restart` + a behavior note in `residual_checks` | +| Repeatable staging access | `README.md` documents setup, execution, diagnostics, and cleanup end to end | `README.md`, `setup.sh`, `run.sh` | +| Redacted evidence only | `state.json` never stores credentials; `evidence.md` is rendered only from non-secret fields | `state.py`, `skill/scripts/render-report.py` | + +See the design doc for the full rationale, including why this is a manual +smoke rather than an automated E2E test, and why the probe reuses +`tests/e2e/baseline`'s toolkit instead of hand-rolling REST/WS calls. diff --git a/examples/sandbox/staging-smoke/skill/scripts/preflight.py b/examples/sandbox/staging-smoke/skill/scripts/preflight.py new file mode 100644 index 000000000..db22fb51f --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/scripts/preflight.py @@ -0,0 +1,105 @@ +""" +Preflight checks for the sandbox staging smoke: tools, environment, and +non-production guardrails — reported without ever printing secret values. + +Reuses `probe.py`'s settings loader (repo-root import, production-URL guard) +rather than re-implementing it; see `probe.py`'s docstring for why both it +and this script reach into `tests/e2e/baseline`. `setup.sh` runs this script +as its own first step rather than duplicating any of these checks itself. + +Exit code 0 = every check passed; 1 = at least one failed (reason on stderr, +no printed secrets). +""" + +from __future__ import annotations + +import logging +import shutil +import subprocess +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + +import root # noqa: F401 (bootstraps sys.path as a side effect) + +import probe +import state + +# No basicConfig call here: importing `probe` above already configured the +# root logger (stderr, "%(asctime)s %(levelname)s %(name)s: %(message)s") — +# a second basicConfig call is a silent no-op once handlers exist, so this +# reuses that configuration rather than pretending to set its own. +logger = logging.getLogger(__name__) + + +class SandboxSettings(BaseSettings): + """The `sbx`-specific config this smoke needs, read the same way + `BaselineSettings` reads Band platform config — not a directory-scoped + `.env`; by the time this constructs, `tests.e2e.baseline.settings` (via + the `probe` import above) has already loaded the repo root's `.env.test`. + """ + + model_config = SettingsConfigDict(extra="ignore", case_sensitive=False) + + sbx_sandbox: str = "" # SBX_SANDBOX + sbx_workspace: str = "" # SBX_WORKSPACE + + +def check(name: str, ok: bool, detail: str = "") -> bool: + status = "PASS" if ok else "FAIL" + suffix = f" — {detail}" if detail else "" + logger.info("[%s] %s%s", status, name, suffix) + return ok + + +def main() -> int: + results: list[bool] = [] + + sbx_path = shutil.which("sbx") + results.append(check("sbx installed", sbx_path is not None)) + if sbx_path: + version_lines = subprocess.run( + ["sbx", "version"], capture_output=True, text=True, check=False + ).stdout.splitlines() + results.append( + check( + "sbx version reported", + bool(version_lines), + version_lines[0] if version_lines else "", + ) + ) + + sandbox = SandboxSettings() + results.append(check("SBX_SANDBOX set", bool(sandbox.sbx_sandbox))) + results.append(check("SBX_WORKSPACE set", bool(sandbox.sbx_workspace))) + if sandbox.sbx_workspace: + inside_repo = ( + Path(sandbox.sbx_workspace) + .expanduser() + .resolve() + .is_relative_to(state.repo_root()) + ) + results.append(check("SBX_WORKSPACE is outside this checkout", not inside_repo)) + + try: + probe.load_settings() + results.append( + check( + "Staging endpoints + BAND_API_KEY_USER present and non-production", + True, + ) + ) + except ValueError as error: + results.append( + check( + "Staging endpoints + BAND_API_KEY_USER present and non-production", + False, + str(error), + ) + ) + + return 0 if all(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sandbox/staging-smoke/skill/scripts/record-observation.py b/examples/sandbox/staging-smoke/skill/scripts/record-observation.py new file mode 100644 index 000000000..94cfea10b --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/scripts/record-observation.py @@ -0,0 +1,43 @@ +""" +Record a behavioral observation into `.sandbox-smoke/state.json` — how a +recovery happened (survival, auto-resume or not, what recovery took), which +the evidence report prints alongside the probe verdicts. The skill calls this +instead of mutating state.json inline, for the same reason `record-phase.py` +exists: the schema and the legal check names live in one place +(`state.OBSERVATION_CHECKS`), and free text passes through as an argument +instead of being spliced into quoted Python inside quoted shell. + +Usage: + record-observation.py +""" + +from __future__ import annotations + +import logging +import sys + +import root # noqa: F401 (bootstraps sys.path as a side effect) + +import state + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +def main() -> int: + if len(sys.argv) < 3 or sys.argv[1] not in state.OBSERVATION_CHECKS: + valid = ", ".join(state.OBSERVATION_CHECKS) + logger.error("Usage: record-observation.py <%s> ", valid) + return 1 + + check = sys.argv[1] + observation = " ".join(sys.argv[2:]) + run_state = state.load() + run_state.residual_checks[check] = observation + state.save(run_state) + logger.info("Recorded %s observation.", check) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sandbox/staging-smoke/skill/scripts/record-phase.py b/examples/sandbox/staging-smoke/skill/scripts/record-phase.py new file mode 100644 index 000000000..99f6efb6a --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/scripts/record-phase.py @@ -0,0 +1,47 @@ +""" +Record a phase transition into `.sandbox-smoke/state.json`, printing that +phase's operator-facing status message (`state.PHASE_MESSAGES`, the single +source of truth). The skill calls this at each checkpoint instead of writing +to state.json directly, so the schema and the phase-to-message mapping live +in one place. + +Usage: + record-phase.py + +The very first call (typically `record-phase.py started`) creates +`.sandbox-smoke/state.json` — or rotates a finished/stale previous run to a +fresh one (`state.load_or_create`); every later call updates the current run. +This is also safe to call after `probe.py --label provision` has already +created the state (the plain operator workflow in README.md mostly doesn't +use this script) — whichever runs first mints the run_id, the other reuses it. +""" + +from __future__ import annotations + +import logging +import sys + +import root # noqa: F401 (bootstraps sys.path as a side effect) + +import state + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +def main() -> int: + if len(sys.argv) != 2 or sys.argv[1] not in state.PHASE_MESSAGES: + valid = ", ".join(state.PHASE_MESSAGES) + logger.error("Usage: record-phase.py <%s>", valid) + return 1 + + phase = sys.argv[1] + run_state = state.load_or_create() + run_state.phase = phase # type: ignore[assignment] # validated by the membership check above + state.save(run_state) + logger.info(state.PHASE_MESSAGES[phase]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sandbox/staging-smoke/skill/scripts/render-report.py b/examples/sandbox/staging-smoke/skill/scripts/render-report.py new file mode 100644 index 000000000..dc2a40b83 --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/scripts/render-report.py @@ -0,0 +1,127 @@ +""" +Render `.sandbox-smoke/evidence.md` from the recorded run state — the +handoff artifact the "Evidence report contract" in the design doc specifies. +Reads only non-secret fields from state.json; never touches credentials. +""" + +from __future__ import annotations + +import logging + +import root # noqa: F401 (bootstraps sys.path as a side effect) + +import state + +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + +# The full run's probe-verified requirements, in workflow order. Every row is +# mandatory: a missing probe renders NOT RUN and makes the overall result +# INCOMPLETE — the full run has no optional checks. +PROBE_REQUIREMENTS: list[tuple[str, str]] = [ + ("initial", "Initial WS receive + REST reply"), + ("after-wifi-reconnect", "Reconnect after Wi-Fi cycle"), + ("after-sleep-wake", "Recovery after host sleep/wake"), + ("after-daemon-restart", "Recovery after sandbox daemon restart"), +] + + +def _requirement_row( + label: str, requirement: str, latest: state.ProbeResult | None +) -> str: + """One scorecard row from the *latest* attempt for ``label`` only — a + probe that failed once and passed on retry must read as PASS, not FAIL + (see `SmokeState.latest`); earlier attempts are history, not the verdict. + """ + if latest is None: + return f"| {requirement} | NOT RUN | no probe recorded |" + status = "PASS" if latest.passed else "FAIL" + return f"| {requirement} | {status} | {label}: marker {latest.marker} |" + + +def render(run_state: state.SmokeState) -> str: + latest_by_label = { + label: run_state.latest(label) for label, _ in PROBE_REQUIREMENTS + } + any_failed = any( + latest is not None and not latest.passed for latest in latest_by_label.values() + ) + any_missing = any(latest is None for latest in latest_by_label.values()) + + # The verdict is derived from the probes alone (plus an explicit operator + # `failed` marking) — never from progress phases, which announce workflow + # position and cannot fabricate or veto a result. + if run_state.phase == "failed" or any_failed: + overall = "FAIL" + elif any_missing: + overall = "INCOMPLETE" + else: + overall = "PASS" + + duration = run_state.updated_at - run_state.started_at + + lines = [ + f"# Sandbox staging smoke — {run_state.run_id}", + "", + "## Result", + "", + f"{overall} — phase: {run_state.phase}", + "", + "## Environment", + "", + f"- Duration: {duration}", + f"- SDK version: {run_state.sdk_version or 'unknown'}", + f"- sbx version: {run_state.sbx_version or 'unknown'}", + f"- Sandbox name: {run_state.sandbox_name or 'unknown'}", + "", + "## Requirement scorecard", + "", + "| Requirement | Status | Evidence |", + "|---|---|---|", + ] + lines += [ + _requirement_row(label, requirement, latest_by_label[label]) + for label, requirement in PROBE_REQUIREMENTS + ] + + # Behavioral observations that accompany the probes (e.g. whether the + # process/VM survived an interruption and what recovery took). These + # describe *how* a recovery happened; the probe rows above are the + # pass/fail verdicts. Every expected check renders — a missing note is + # visibly "(not recorded)" rather than silently absent from a PASS report. + lines += ["", "## Observed behavior", ""] + for check_name in state.OBSERVATION_CHECKS: + observation = run_state.residual_checks.get(check_name, "(not recorded)") + lines.append(f"- **{check_name}**: {observation}") + for check_name, observation in run_state.residual_checks.items(): + if check_name not in state.OBSERVATION_CHECKS: + lines.append(f"- **{check_name}**: {observation}") + + lines += ["", "## Timeline", ""] + for probe in run_state.probes: + detail = f" ({probe.detail})" if probe.detail else "" + lines.append( + f"- {probe.at.isoformat()} — {probe.label}: " + f"{'PASS' if probe.passed else 'FAIL'}{detail}" + ) + + lines += [ + "", + "## Follow-up", + "", + "(none)" if overall == "PASS" else "See probe details above.", + ] + return "\n".join(lines) + "\n" + + +def main() -> int: + run_state = state.load() + report = render(run_state) + path = state.state_dir() / "evidence.md" + path.write_text(report, encoding="utf-8") + logger.info("Wrote %s", path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/sandbox/staging-smoke/skill/scripts/root.py b/examples/sandbox/staging-smoke/skill/scripts/root.py new file mode 100644 index 000000000..5e2cf5265 --- /dev/null +++ b/examples/sandbox/staging-smoke/skill/scripts/root.py @@ -0,0 +1,21 @@ +""" +Locate `examples/sandbox/staging-smoke/` from any script under `skill/scripts/` +and put it on `sys.path`, so sibling scripts can `import state` / `import probe`. + +Fixed by this skill's own layout (`skill/scripts/` always sits exactly two +directories below the example root) — not a generic walk-up-for-a-marker +search, which would solve a problem this specific, versioned directory layout +doesn't have (and would duplicate `state.py`'s own, differently-scoped +`repo_root()`). Import this before anything else: `import root # noqa: F401`. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +STAGING_SMOKE_ROOT = ( + Path(__file__).resolve().parents[2] +) # scripts -> skill -> example root +if str(STAGING_SMOKE_ROOT) not in sys.path: + sys.path.insert(0, str(STAGING_SMOKE_ROOT)) diff --git a/examples/sandbox/staging-smoke/state.py b/examples/sandbox/staging-smoke/state.py new file mode 100644 index 000000000..53f04a12c --- /dev/null +++ b/examples/sandbox/staging-smoke/state.py @@ -0,0 +1,269 @@ +""" +Shared, gitignored run-state for the Docker Sandbox staging smoke. + +One `SmokeState` record lives at `.sandbox-smoke/state.json` for the lifetime +of a single operator run, persisted across the run's operator checkpoints — +the processes exit at the Wi-Fi and sleep/wake handoffs and during the +daemon-restart recovery, so state must survive on disk. `probe.py` and the +`skill/scripts/*.py` orchestration both read/write through this module so the +schema and file location live in exactly one place. + +Never stores secrets: only ids, names, timestamps, and phase/result fields. +Each step re-reads credentials (BAND_API_KEY_USER, the provisioned agent's +api_key) from the approved secret source instead of persisting them here. + +Ownership: `phase` is written by `record-phase.py`, plus exactly one write in +`probe.py` — terminal `cleanup` marks `completed`, the shared end-of-run for +the plain operator workflow, which never calls `record-phase.py`. `probes` +and the resource ids are written by `probe.py`; `residual_checks` by +`record-observation.py`. The report verdict is derived from `probes`, never +from `phase` (see `render-report.py`), so a stale or mid-run phase can never +fabricate a result. + +This module (like every other file in this directory except `agent.py`) has no +PEP 723 header: it's imported by scripts that run inside this repository's own +dev environment (`uv sync --extra dev`), not executed standalone, and it needs +no third-party dependency beyond what that environment already provides. +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Literal, get_args + +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +# Every phase is load-bearing — a checkpoint breadcrumb, a recovery marker, +# or a terminal state; progress narration is the orchestrator's job, not the +# state machine's: +# started — the run exists (rotation boundary for a new run) +# awaiting-wifi-recovery — operator checkpoint; resume instruction +# awaiting-sleep-wake — operator checkpoint; resume instruction +# daemon-restart — marks the one mid-run case where re-provisioning +# under the same run is legitimate (begin_provision) +# completed / failed — terminal; the next entry point starts a fresh run +Phase = Literal[ + "started", + "awaiting-wifi-recovery", + "awaiting-sleep-wake", + "daemon-restart", + "completed", + "failed", +] + +# Phases that end a run. A later entry point must start a fresh run rather +# than silently continue a finished one (see load_or_create). +TERMINAL_PHASES: frozenset[str] = frozenset({"completed", "failed"}) + +# An in-flight run older than this is treated as abandoned, not resumable: +# rotating to a fresh run keeps its stale probes from leaking into a new +# report. Generous enough for a long lunch mid-checkpoint, far shorter than +# "operator came back next week". +MAX_RESUMABLE_AGE = timedelta(hours=12) + +# The behavioral observations the full run must record (written via +# record-observation.py, rendered by render-report.py — both reference this +# list, so a check can't be named in one place and not the other). +OBSERVATION_CHECKS: tuple[str, ...] = ("sleep_wake", "daemon_restart") + +# The one required status message per phase — the single source of truth +# `record-phase.py` prints from, so a phase can never be added without its +# operator-facing message (enforced by the assert below). +PHASE_MESSAGES: dict[Phase, str] = { + "started": "Starting the sandbox staging smoke.", + "awaiting-wifi-recovery": ( + "Turn Wi-Fi off, wait for the agent log to show disconnect, turn it " + "back on, wait for reconnect, then ask to resume this run." + ), + "awaiting-sleep-wake": ( + "Put the host to sleep for about a minute, wake it, then ask to " + "resume this run." + ), + "daemon-restart": ( + "Restarting the sandbox daemon; expect the agent process to die and " + "be re-provisioned." + ), + "completed": "Smoke complete; see .sandbox-smoke/evidence.md.", + "failed": "Smoke failed; see .sandbox-smoke/evidence.md for details.", +} +assert set(PHASE_MESSAGES) == set(get_args(Phase)), ( + "PHASE_MESSAGES and Phase have drifted — every phase needs exactly one message" +) + + +class ProbeResult(BaseModel): + label: str + marker: str + passed: bool + detail: str = "" + at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class SmokeState(BaseModel): + """Non-secret run state persisted across the run's operator checkpoints.""" + + run_id: str + phase: Phase = "started" + sandbox_name: str = "" + sbx_version: str = "" + sdk_version: str = "" + room_id: str = "" + agent_id: str = "" + agent_name: str = "" + probes: list[ProbeResult] = Field(default_factory=list) + # Free-text behavioral observations, one per OBSERVATION_CHECKS entry + # (how a recovery happened — not a verdict; the probes are the verdicts). + residual_checks: dict[str, str] = Field(default_factory=dict) + started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + def latest(self, label: str) -> ProbeResult | None: + """The most recent attempt for ``label`` — a probe may be retried after + a transient failure, and only the latest attempt should count toward + PASS/FAIL (earlier failed attempts are history, not the verdict).""" + matching = [p for p in self.probes if p.label == label] + return matching[-1] if matching else None + + def is_resumable(self) -> bool: + """True when a new entry point should continue this run rather than + start a fresh one: the run is neither finished nor abandoned. Without + this boundary, a later run would inherit this run's probes and could + render a report containing results that were never re-proven.""" + if self.phase in TERMINAL_PHASES: + return False + updated = self.updated_at + if updated.tzinfo is None: + updated = updated.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) - updated < MAX_RESUMABLE_AGE + + +def root_dir() -> Path: + return Path(__file__).resolve().parent + + +def repo_root() -> Path: + """The band-sdk-python repo root. + + Fixed by this example's own location in the tree + (`examples/sandbox/staging-smoke/`, always exactly two directories below + the repo root) — not a generic walk-up-for-a-marker search, which would + solve a problem this specific, versioned directory layout doesn't have. + """ + return root_dir().parents[2] + + +def state_dir() -> Path: + return root_dir() / ".sandbox-smoke" + + +def state_path() -> Path: + return state_dir() / "state.json" + + +def load() -> SmokeState: + path = state_path() + if not path.exists(): + raise FileNotFoundError( + f"No run state at {path}. Run setup.sh (or the skill's preflight) first." + ) + return SmokeState.model_validate_json(path.read_text(encoding="utf-8")) + + +def save(run_state: SmokeState) -> None: + """Write ``run_state`` atomically: a write killed mid-flight (Ctrl-C, the + operator's Wi-Fi toggle) must never leave a truncated, unloadable + state.json — the whole point of this being a resumable workflow.""" + run_state.updated_at = datetime.now(timezone.utc) + state_dir().mkdir(parents=True, exist_ok=True) + path = state_path() + tmp_path = path.with_suffix(".json.tmp") + tmp_path.write_text(run_state.model_dump_json(indent=2), encoding="utf-8") + tmp_path.chmod(0o600) + tmp_path.replace(path) + + +def new(run_id: str | None = None) -> SmokeState: + run_state = SmokeState(run_id=run_id or uuid.uuid4().hex[:8]) + save(run_state) + return run_state + + +def load_or_create() -> SmokeState: + """Load the in-flight run, or start a fresh one. + + Two independent entry points share this: `record-phase.py started` (the + AI-orchestrated skill workflow's first step) and `probe.py --label + provision` (the plain operator workflow's first Band-touching step, which + never calls `record-phase.py` at all). Whichever runs first creates the + state; the other reuses it, so both workflows share one run_id. + + A finished or abandoned run is **not** reused (`SmokeState.is_resumable`): + its file is archived beside state.json (evidence preserved, never + silently continued) and a fresh run begins. Only a fresh, in-flight run — + e.g. the daemon-restart recovery re-provision — resumes. + """ + if not state_path().exists(): + return new() + existing = load() + if existing.is_resumable(): + return existing + return _rotate(existing) + + +def begin_provision() -> SmokeState: + """The run a `probe.py --label provision` should provision into. + + Reuses the current run only when it can't fabricate evidence: either no + probes have been recorded yet (the normal first provision, or a retry + after a crashed launch), or the workflow explicitly marked a + daemon-restart recovery (`phase == "daemon-restart"`, set right before + the daemon bounce — the one mid-run case where re-provisioning under the + same run is correct). Any other pre-existing state with recorded probes + is archived and a fresh run begins, so a new smoke can never inherit a + previous run's PASS rows. + """ + run_state = load_or_create() + if run_state.probes and run_state.phase != "daemon-restart": + return _rotate(run_state) + return run_state + + +def _rotate(existing: SmokeState) -> SmokeState: + """Archive ``existing``'s file beside state.json and start a fresh run — + evidence is preserved, never silently continued. + + Resource ids the old run never reaped (a failed or abandoned run whose + cleanup never ran) are carried into the fresh run, not dropped with the + archive: the current state.json is the only record the reap paths read, + and nothing else ever deletes a room (the orphan sweep covers only aged + agents). The next `provision` reaps carried ids before provisioning + replacements, and `cleanup` retries them. A cleanly finished run carries + nothing — cleanup clears its ids before marking it terminal. + """ + archive = state_dir() / f"state.{existing.run_id}.json" + state_path().replace(archive) + logger.info( + "Previous run %s is %s; archived to %s and starting a fresh run", + existing.run_id, + "finished" if existing.phase in TERMINAL_PHASES else "stale or superseded", + archive, + ) + fresh = new() + if existing.room_id or existing.agent_id: + fresh.room_id = existing.room_id + fresh.agent_id = existing.agent_id + fresh.agent_name = existing.agent_name + save(fresh) + logger.info( + "Run %s left unreaped resources (room=%s, agent=%s); carried " + "into the new run for the next provision/cleanup to reap", + existing.run_id, + existing.room_id or "-", + existing.agent_id or "-", + ) + return fresh diff --git a/tests/e2e/baseline/fixtures/platform.py b/tests/e2e/baseline/fixtures/platform.py index 69c8653a4..46d2546b9 100644 --- a/tests/e2e/baseline/fixtures/platform.py +++ b/tests/e2e/baseline/fixtures/platform.py @@ -10,15 +10,14 @@ import pytest from band_rest import AsyncRestClient -from band.client.streaming import WebSocketClient - from tests.e2e.baseline.settings import BaselineSettings from tests.e2e.baseline.toolkit.provisioning import ( ResourceManager, new_run_id, + user_rest_client, ) from tests.e2e.baseline.toolkit.user_ops import UserOps -from tests.e2e.baseline.toolkit.ws import TrackingWebSocketClient +from tests.e2e.baseline.toolkit.ws import TrackingWebSocketClient, user_ws_observer logger = logging.getLogger(__name__) @@ -48,13 +47,7 @@ def baseline_run_id() -> str: @pytest.fixture(scope="session") def baseline_user_client(baseline_settings: BaselineSettings) -> AsyncRestClient: """Session-scoped user-authenticated REST client for provisioning.""" - assert baseline_settings.credentials.api_key_user, ( - "BAND_API_KEY_USER is required for provisioning" - ) - return AsyncRestClient( - api_key=baseline_settings.credentials.api_key_user, - base_url=baseline_settings.endpoints.rest_url, - ) + return user_rest_client(baseline_settings) @pytest.fixture @@ -73,19 +66,11 @@ async def baseline_ws( ) -> AsyncGenerator[TrackingWebSocketClient, None]: """User-authenticated WS observer for the wait primitives. - Connects as the user (not an agent), so it coexists with agents and - receives the same ``message_created`` events. Session-scoped to avoid - per-test connect/teardown latency; channels are left on teardown. + Session-scoped to avoid per-test connect/teardown latency; channels are + left on teardown. Construction lives in ``user_ws_observer`` (shared with + pytest-free callers). """ - assert baseline_settings.credentials.api_key_user, ( - "BAND_API_KEY_USER is required for the WS observer" - ) - ws = WebSocketClient( - ws_url=baseline_settings.endpoints.ws_url, - api_key=baseline_settings.credentials.api_key_user, - agent_id=None, # user connection, not an agent - ) - async with ws, TrackingWebSocketClient(ws) as tracking: + async with user_ws_observer(baseline_settings) as tracking: yield tracking diff --git a/tests/e2e/baseline/toolkit/provisioning.py b/tests/e2e/baseline/toolkit/provisioning.py index ac61eebb7..d936d1495 100644 --- a/tests/e2e/baseline/toolkit/provisioning.py +++ b/tests/e2e/baseline/toolkit/provisioning.py @@ -100,6 +100,23 @@ class ProvisionedAgent: adapter_id: str | None = None +def user_rest_client(settings: BaselineSettings) -> AsyncRestClient: + """A user-authenticated REST client — the test-driver/observer identity. + + The one construction of the user client, shared by the pytest fixture + (``baseline_user_client``) and pytest-free callers (e.g. the sandbox + staging smoke's ``probe.py``), so the two can never drift. Like + ``agent_rest_client`` below, the Fern client wraps an httpx pool with no + public close hook and is left to be reclaimed at event-loop teardown. + """ + if not settings.credentials.api_key_user: + raise ValueError("BAND_API_KEY_USER is required for the user REST client") + return AsyncRestClient( + api_key=settings.credentials.api_key_user, + base_url=settings.endpoints.rest_url, + ) + + def agent_rest_client( agent: ProvisionedAgent, settings: BaselineSettings ) -> AsyncRestClient: diff --git a/tests/e2e/baseline/toolkit/ws.py b/tests/e2e/baseline/toolkit/ws.py index 22e402772..48467cad0 100644 --- a/tests/e2e/baseline/toolkit/ws.py +++ b/tests/e2e/baseline/toolkit/ws.py @@ -4,18 +4,45 @@ it has joined so they can all be left on teardown. Only the methods the toolkit uses are explicitly delegated — no ``__getattr__`` proxy — so typos surface as type-checker errors instead of silent runtime failures. + +``user_ws_observer`` is the one construction of the user-authenticated +observer connection, shared by the pytest fixture (``baseline_ws``) and +pytest-free callers (e.g. the sandbox staging smoke's ``probe.py``). """ from __future__ import annotations import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager from band.client.streaming import MessageCreatedPayload, WebSocketClient +from tests.e2e.baseline.settings import BaselineSettings + logger = logging.getLogger(__name__) +@asynccontextmanager +async def user_ws_observer( + settings: BaselineSettings, +) -> AsyncIterator[TrackingWebSocketClient]: + """Connect a user-authenticated WS observer; leave its channels on exit. + + Connects as the user (not an agent), so it coexists with agents and + receives the same ``message_created`` events. + """ + if not settings.credentials.api_key_user: + raise ValueError("BAND_API_KEY_USER is required for the WS observer") + ws = WebSocketClient( + ws_url=settings.endpoints.ws_url, + api_key=settings.credentials.api_key_user, + agent_id=None, # user connection, not an agent + ) + async with ws, TrackingWebSocketClient(ws) as tracking: + yield tracking + + class TrackingWebSocketClient: """Async-context-manager wrapper that tracks joined rooms and leaves them on exit.