diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6309198..7ce725e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,9 +10,70 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Both lanes below are offline: the LLM boundary is served from the committed + # cassette store. That is deliberate — a pull request from a fork cannot access + # secrets, so anything gating a merge must run without them. + LEAPFLOW_TEST_LLM_MODE: replay + jobs: - test: + # ── L1: pull-request lane ────────────────────────────────────────────── + # Static checks, the whole real layer, and the whole mock layer. + # + # Change-scoped selection is deliberately NOT used here, and the reason is + # arithmetic rather than distrust. Measured on this suite: the full mock layer + # is ~18s, while the always-on tier alone (real journeys + regression ledger + + # architecture contracts) is ~14s. Those tiers can never be selected away, so + # they set the floor — selecting the mock layer can save at most ~4s, however + # precise the selection gets. Spending correctness risk on four seconds is a + # bad trade. + # + # Revisit when the full mock layer stops fitting the feedback budget — + # concretely, when `make test-unit` exceeds ~3 minutes on CI hardware. The + # machinery is ready and tested (tools/impact.py, with a coverage-derived map + # in tests/.impact/); `make test-impact` already uses it locally, where a + # single-module change narrows to 2-3 test files. + # + # The live lane *does* select, in .github/workflows/nightly-live.yaml: there a + # journey costs real tokens, so the arithmetic comes out the other way. + pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --all-extras + + - name: Lint + run: uv run ruff check src/leapflow/ tests/ tools/ + + - name: Verify derived fixtures match the cassette store + run: uv run python tools/sync_fixtures.py --check + + - name: Real layer — journeys and always-on guards + run: uv run pytest tests/journeys tests/regression tests/test_architecture_contracts.py -q -m "e2e or invariant or unit" -n 4 + + - name: Mock layer (full) + run: uv run pytest tests/ -q -m "not e2e" --tb=short -n auto + + # ── L2: main lane ────────────────────────────────────────────────────── + # Everything, unscoped, across the supported matrix. + main: + if: github.event_name == 'push' runs-on: ${{ matrix.os }} + timeout-minutes: 40 strategy: fail-fast: false matrix: @@ -29,15 +90,29 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 + with: + enable-cache: true - name: Install dependencies run: uv sync --all-extras - name: Lint - run: uv run ruff check src/leapflow/ tests/ + run: uv run ruff check src/leapflow/ tests/ tools/ + + - name: Verify derived fixtures match the stored exchanges + run: uv run python tools/sync_fixtures.py --check + + - name: Mock layer — full + run: uv run pytest tests/ -q -m "not e2e" --tb=short -n auto + + - name: Real layer — full + run: uv run pytest tests/journeys -q -m e2e --tb=short -n 4 - - name: Run tests - run: uv run pytest tests/ -q --tb=short - env: - LEAPFLOW_MOCK_HOST: '1' - LEAPFLOW_LLM_API_KEY: 'test-key-ci' + - name: Daemon logs on failure + if: failure() + run: | + echo "Journey daemons log under the scratch root; surface anything left behind." + find /tmp -maxdepth 6 -name 'leapd.log' -newermt '-40 minutes' 2>/dev/null | while read -r log; do + echo "===== $log =====" + tail -n 120 "$log" + done diff --git a/.github/workflows/nightly-live.yaml b/.github/workflows/nightly-live.yaml new file mode 100644 index 0000000..a8748a0 --- /dev/null +++ b/.github/workflows/nightly-live.yaml @@ -0,0 +1,213 @@ +name: Nightly live + +# The only lane that talks to a real provider. It exists to catch what replay +# structurally cannot: a provider changing its behavior or its payload shape. +# Everything else runs offline, so a red build here never blocks a merge — it +# tells us the recorded truth has drifted from the real one. + +on: + schedule: + # 02:30 UTC daily. + - cron: '30 2 * * *' + workflow_dispatch: + inputs: + rerecord: + description: 'Capture fresh provider traffic and open a PR with it' + type: boolean + default: false + # Opt-in per pull request via the `ci:live` label. Unlike the schedule, this + # path has a diff, so it runs only the journeys the change could plausibly + # break — each live journey costs real tokens and real minutes. + pull_request: + types: [labeled, synchronize, reopened] + +concurrency: + group: nightly-live-${{ github.event.pull_request.number || 'schedule' }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + # ── L3: real provider ────────────────────────────────────────────────── + live: + # On a pull request, only with the `ci:live` label — never automatically, so a + # fork PR cannot spend tokens. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'ci:live') + runs-on: ubuntu-latest + timeout-minutes: 40 + # Credentials live as secrets on this environment, so only jobs that declare + # it can read them. Deliberately *without* required-reviewer or + # deployment-branch rules: reviewers would leave the nightly cron waiting for + # a human, and restricting branches to main would reject every `ci:live` run + # (a pull_request ref is refs/pull/N/merge). The real gates are that fork PRs + # never receive secrets, that applying the label needs write access, and that + # each journey caps its own calls and tokens. + environment: live-llm + steps: + - uses: actions/checkout@v4 + with: + # Journey selection needs history to find the merge base. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --all-extras + + - name: Decide which journeys to run + id: pick + # A scheduled run has no diff and takes every live-capable journey. A + # labelled pull request takes only the journeys whose declared + # SUBJECT_PATHS the change touches. Journeys with LIVE_SIGNAL = False + # (control plane, lifecycle) are excluded either way, and R4 additionally + # refuses to run live because it asserts on injected failures. + env: + BASE_REF: ${{ github.base_ref }} + run: | + if [ -n "${BASE_REF}" ]; then + JOURNEYS=$(uv run python tools/impact.py --base "origin/${BASE_REF}" --live-journeys) + else + JOURNEYS=$(uv run python tools/impact.py --live-journeys) + fi + printf 'selected journeys:\n%s\n' "${JOURNEYS}" + echo "journeys=$(echo ${JOURNEYS} | tr '\n' ' ')" >> "$GITHUB_OUTPUT" + + - name: Journeys against the real provider + if: steps.pick.outputs.journeys != '' + env: + LEAPFLOW_TEST_LLM_MODE: live + LEAPFLOW_LLM_API_KEY: ${{ secrets.LEAPFLOW_LLM_API_KEY }} + LEAPFLOW_LLM_BASE_URL: ${{ secrets.LEAPFLOW_LLM_BASE_URL }} + # A cheap model keeps the lane affordable; the journeys assert + # invariants, not prose quality. Each journey also enforces its own + # provider-call *and* token ceilings, so neither a non-converging turn + # nor prompt growth can run up a bill. + LEAPFLOW_LLM_MODEL: ${{ secrets.LEAPFLOW_LLM_CHEAP_MODEL }} + JOURNEYS: ${{ steps.pick.outputs.journeys }} + run: uv run pytest ${JOURNEYS} -q -m e2e --tb=short + + - name: Daemon logs on failure + if: failure() + run: | + find /tmp -maxdepth 6 -name 'leapd.log' -newermt '-40 minutes' 2>/dev/null | while read -r log; do + echo "===== $log =====" + tail -n 200 "$log" + done + + # ── Re-record: refresh recorded truth and propose it as a diff ────────── + # Manual only. Recorded traffic is a reviewed artefact: a bot silently updating + # what the mock layer asserts against would defeat the point of recording it. + # Recording writes to recordings/ and never touches the replay store, so this + # job cannot break the offline lanes. + rerecord: + if: github.event_name == 'workflow_dispatch' && inputs.rerecord == true + runs-on: ubuntu-latest + timeout-minutes: 40 + environment: live-llm + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --all-extras + + - name: Capture real provider traffic + env: + LEAPFLOW_TEST_LLM_MODE: record + LEAPFLOW_LLM_API_KEY: ${{ secrets.LEAPFLOW_LLM_API_KEY }} + LEAPFLOW_LLM_BASE_URL: ${{ secrets.LEAPFLOW_LLM_BASE_URL }} + LEAPFLOW_LLM_MODEL: ${{ secrets.LEAPFLOW_LLM_CHEAP_MODEL }} + run: uv run pytest tests/journeys -q -m e2e --tb=short + + - name: Derive mock-layer response shapes from the new traffic + run: uv run python tools/sync_fixtures.py + + - name: Confirm the offline lanes still pass + env: + LEAPFLOW_TEST_LLM_MODE: replay + run: uv run pytest tests/journeys tests/regression -q -m "e2e or invariant" -n 4 + + - name: Open a pull request with the refreshed traffic + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/rerecord-provider-traffic + title: 'chore(tests): refresh recorded provider traffic' + body: | + Captured fresh provider traffic and re-derived the response shapes the + mock layer checks against. + + Review the diff in `tests/_fixtures/llm_responses/response_shapes.json` + first: a change there means a provider altered its payload shape, and + some parser may now be reading a field that no longer exists. + commit-message: 'chore(tests): refresh recorded provider traffic' + add-paths: | + tests/_fixtures/recordings/** + tests/_fixtures/llm_responses/** + + # ── Refresh the impact map from a full green run ──────────────────────── + impact-map: + # Never on a pull request: the map is a repository artefact refreshed from a + # full green run, not something a PR should regenerate. + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --all-extras + + - name: Rebuild the coverage-derived impact map + env: + LEAPFLOW_TEST_LLM_MODE: replay + run: uv run python tools/impact.py --build-map + + - name: Open a pull request with the refreshed map + uses: peter-evans/create-pull-request@v6 + with: + branch: chore/refresh-impact-map + title: 'chore(tests): refresh the coverage-derived impact map' + body: | + Regenerated `tests/.impact/coverage_map.json` from a full green run. + + This map is what lets the pull-request lane scope the mock layer to + the change while still seeing runtime coupling through EventBus and + Protocol indirection. + commit-message: 'chore(tests): refresh the coverage-derived impact map' + add-paths: tests/.impact/coverage_map.json diff --git a/AGENTS.md b/AGENTS.md index 15d442c..58f6bb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,12 +118,33 @@ This document is the LeapFlow engineering collaboration contract. It is not only ## Testing Philosophy +The suite has **two layers with different jobs**, and keeping the boundary sharp is what stops each from doing the other's work badly. + +**Mock layer** (`tests/*.py`, marked `unit`/`component`) — broad and fast. It owns pure algorithms, state-machine branches, error-classification tables, rare and malformed inputs, and single-module invariants. Branch combinations can only be enumerated here, and only here is the feedback measured in milliseconds. + +**Real layer** (`tests/journeys/`, marked `e2e`) — six coarse journeys driving a real `leapd` subprocess over RPC, with the LLM boundary served by a local cassette proxy. It owns what a mock structurally cannot observe: cross-module wiring, process boundaries, session identity, workspace binding, real persistence, and pushed runtime metadata. Every incident recorded in this document shipped with a green mock suite. + +Three rules keep the split honest: + +1. **Anything provable with one mock-layer assertion must not enter the real layer.** +2. **One journey is one test case.** Express variation as ordered phases inside a single session; never parameterize a journey. +3. **The real layer has a hard case budget** (`tests/regression/test_suite_budget.py`). When it is reached, merge a journey — do not raise the ceiling. The budget is what lets the real layer run on *every* push, and a suite that can be skipped will be skipped. + +The two layers are joined by recorded traffic. `tests/_fixtures/cassettes/` holds the deterministic inputs the offline lanes replay (rebuilt with `make seed-cassettes`); `tests/_fixtures/recordings/` holds real provider traffic captured by `make record-traffic`. `make sync-fixtures` distils both into `tests/_fixtures/llm_responses/response_shapes.json`, which the mock layer asserts against — so a provider dropping a field turns the build red instead of passing forever against a body written from memory. Recording never writes to the replay store: a multi-turn agent conversation cannot be replayed from a recording, because each turn's prompt embeds the exact round-by-round history of the turns before it. + +Each journey also declares two cost ceilings, both enforced at the proxy and reported by `finish()`. `max_llm_calls` is the convergence guard: a turn that stops converging is cut off instead of running to the engine's iteration cap. `max_llm_tokens` is the cost guard, and it catches what call count cannot — prompt growth raises the bill without adding a single round. Raising a ceiling is not the fix for hitting one. + +**Which tests run.** The offline lanes never select: the mock layer runs in full, and the real journeys run in full on every push. Selection would save at most a few seconds, because the always-on tiers set the floor, and a suite that can be skipped will be skipped. The *live* lane is the exception — there a journey costs real tokens, so it selects: each journey declares `SUBJECT_PATHS` (the sources it exercises) and `LIVE_SIGNAL` (whether a real provider adds signal), and `tools/impact.py --live-journeys` picks from those. Declaring them inside the journey keeps that knowledge next to the assertions it describes. `tools/impact.py` can also scope the mock layer (`make test-impact`) for local work on a large change; it is not wired into CI. + - **Unit tests must be hermetic**: no network, no LLM calls +- **Journeys must not mock anything**: a journey that reaches for `unittest.mock` has become an expensive unit test +- **Provider bodies are recorded, not written**: use a cassette or a derived fixture; a hand-authored body keeps passing after the provider changes +- **Faked construction needs real-instance cover**: `object.__new__` plus private-attribute assignment is acceptable only for ordering contracts, and only when the same file also builds the class properly and drives the production path - **py_compile all modified files**: syntax errors caught before test run - **Import chain verification**: every new module must be importable standalone - **Existing tests must not regress**: all tests must pass after every change - **User-facing flows must not regress**: preserve or improve usability, feedback clarity, and failure recovery for impacted paths -- **Verification sequence**: compile → import → unit test → integration (if applicable) +- **Verification sequence**: compile → import → mock layer → real journeys - **Behavior contracts over snapshots**: assert invariants, not frozen values - **Mock at boundaries only**: mock external I/O (network, disk), never internal logic - **A test may not fabricate the wiring it claims to cover**: building an object with `object.__new__` and assigning the private attributes the code reads cannot detect a wrong attribute *name* — the test simply agrees with the typo. Calibration tests did exactly that and stayed green while every real turn raised `AttributeError`. Any test whose stated purpose is wiring must construct the real object and drive the production path. diff --git a/Makefile b/Makefile index 1a79702..8bd70f6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,17 @@ # ── Variables ───────────────────────────────────────────────────────────────── LEAPFLOW_DATA_DIR ?= $(HOME)/.leapflow -.PHONY: setup sync test brain lint cua-check +# Change-scoped selection compares against this ref (see tools/impact.py). +BASE ?= origin/main +# Parallelism for the mock layer. The real layer runs at -n 4 (few, heavy cases). +JOBS ?= auto + +.PHONY: setup sync test test-unit test-e2e test-live test-impact test-full \ + record-traffic seed-cassettes sync-fixtures lint brain cua-check help + +help: ## Show available targets + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' setup: ## Setup scripts permissions and environment chmod +x scripts/setup.sh scripts/run.sh @@ -10,13 +20,47 @@ setup: ## Setup scripts permissions and environment sync: ## Sync all dependencies uv sync --all-extras -test: ## Run tests - uv run pytest tests/ -q - lint: ## Lint source code - uv run ruff check src/leapflow/ tests/ + uv run ruff check src/leapflow/ tests/ tools/ + +# ── Test layers ─────────────────────────────────────────────────────────────── +# The mock layer is broad and fast; the real layer is small, coarse, and never +# skipped. Both run offline: the LLM boundary is served from committed cassettes. + +test: test-unit test-e2e ## Default gate: mock layer + real journeys (offline) + +test-unit: ## Mock layer — hermetic units and components + uv run pytest tests/ -q -m "not e2e" -n $(JOBS) + +test-e2e: ## Real layer — coarse journeys against a real leapd, cassette replay + uv run pytest tests/journeys tests/regression -q -m "e2e or invariant" -n 4 + +test-full: ## Everything, unselected + uv run pytest tests/ -q -n $(JOBS) + +test-impact: ## Mock layer scoped to what changed since $(BASE), plus always-on tiers + @uv run python tools/impact.py --base $(BASE) --run + +test-live: ## Real layer against a real provider (needs LEAPFLOW_LLM_* credentials) + LEAPFLOW_TEST_LLM_MODE=live uv run pytest tests/journeys -q -m e2e + +# ── Recorded provider traffic ───────────────────────────────────────────────── +# Two stores, two jobs. `cassettes/` holds the deterministic inputs the offline +# lanes replay; `recordings/` holds real provider traffic, which is evidence of +# wire shape rather than a replay input — a multi-turn agent conversation cannot +# be replayed from a recording, because each turn's prompt embeds the exact +# round-by-round history of the turns before it. + +seed-cassettes: ## Rebuild the offline replay store from each journey's script + LEAPFLOW_TEST_LLM_MODE=seed uv run pytest tests/journeys -q -m e2e + +record-traffic: ## Capture real provider traffic into recordings/ (needs credentials) + LEAPFLOW_TEST_LLM_MODE=record uv run pytest tests/journeys -q -m e2e + +sync-fixtures: ## Derive mock-layer response shapes from both stores + uv run python tools/sync_fixtures.py -# LeapFlow CLI (pass PROMPT via ARGS, e.g. make brain ARGS='--prompt "hello"') +# ── LeapFlow CLI (pass PROMPT via ARGS, e.g. make brain ARGS='--prompt "hello"') brain: ## Start Brain process uv run leap $(ARGS) diff --git a/README.md b/README.md index 1912b9c..fdb2d8f 100644 --- a/README.md +++ b/README.md @@ -970,12 +970,57 @@ class MyChannel: ### Running Tests +The suite has two layers. The **mock layer** is broad and fast; the **real layer** is six +coarse journeys that drive an actual `leapd` subprocess over RPC, with the LLM boundary +served from committed recordings. Both run offline. + ```bash -make test # Full suite +make test # Mock layer + real journeys (the default gate) +make test-unit # Mock layer only, parallel +make test-e2e # Real journeys + always-on guards +make test-impact BASE=origin/main # Mock layer scoped to what changed (local) +make test-full # Everything, unscoped + uv run pytest tests/test_pure_algorithms.py -q # Single file uv run pytest -k "test_world_model" -q # By keyword ``` +CI runs both layers in full. Change-scoped selection exists (`tools/impact.py`, +backed by a coverage-derived map in `tests/.impact/`) and narrows a single-module +change to 2–3 test files, but it is a *local* convenience: the always-on tiers +cannot be selected away and already account for most of the run, so selecting in +CI would save only a few seconds. + +The live lane is the exception — there a journey costs real tokens, so it selects. +Each journey declares `SUBJECT_PATHS` and `LIVE_SIGNAL`, and every journey caps +its own provider calls *and* tokens, so neither a non-converging turn nor prompt +growth can run up a bill. + +Against a real provider (needs `LEAPFLOW_LLM_API_KEY`, `LEAPFLOW_LLM_BASE_URL` and +`LEAPFLOW_LLM_MODEL`): + +```bash +make test-live # Journeys against the real provider +make record-traffic # Capture real traffic into recordings/ +make sync-fixtures # Re-derive mock-layer response shapes +``` + +There are two stores, with different jobs: + +- `tests/_fixtures/cassettes/` — deterministic inputs the offline lanes replay. + Rebuild with `make seed-cassettes`. +- `tests/_fixtures/recordings/` — real provider traffic, kept as evidence of wire + shape. `sync-fixtures` distils both into `llm_responses/response_shapes.json`, + which the mock layer checks against, so a provider dropping a field turns the + build red instead of passing forever. + +Recording deliberately never writes to the replay store: a multi-turn agent +conversation cannot be replayed from a recording, because each turn's prompt +embeds the exact round-by-round history of the turns before it. + +When a journey reports a cassette miss it prints a diff against the nearest +recorded request, so you can see which prompt drifted. + ## Key Modules diff --git a/pyproject.toml b/pyproject.toml index 7b1df3f..5d84fb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,10 @@ dev = [ "pytest>=8.0.0", "pytest-asyncio>=0.23.0", "ruff>=0.4.0", + # Parallel execution for the mock-heavy suite, and coverage for the + # impact map that drives change-scoped selection (tools/impact.py). + "pytest-xdist>=3.5", + "pytest-cov>=5.0", ] hub = ["modelscope-hub>=0.1.0"] dashboard = ["aiohttp>=3.9"] @@ -71,6 +75,16 @@ include = ["leapflow*"] asyncio_mode = "auto" pythonpath = ["src", "tests"] testpaths = ["tests"] +# Markers are applied by path in tests/conftest.py, so existing files need no +# edit. Only files that do real local IO opt in explicitly with `pytestmark`. +markers = [ + "unit: hermetic — no real IO, no LLM. Default for tests/*.py", + "component: real local IO (DuckDB, tmp profile) in-process; LLM via cassette replay", + "e2e: real leapd subprocess driven over RPC; LLM via cassette replay", + "live: journey assertions against a real provider (nightly lane only)", + "invariant: always-on guard — never skipped by impact selection", + "slow: takes more than a few seconds", +] [tool.ruff] line-length = 100 diff --git a/src/leapflow/engine/session.py b/src/leapflow/engine/session.py index 1dd5223..567cbbd 100644 --- a/src/leapflow/engine/session.py +++ b/src/leapflow/engine/session.py @@ -143,9 +143,15 @@ def is_distilling(self) -> bool: @property def recording_step_count(self) -> int: - """Number of steps captured so far in the active recording.""" + """Number of steps captured so far in the active recording. + + The count lives on the trajectory the recorder is filling, not on the + recorder itself. Reading a non-existent ``recorder.step_count`` raised + ``AttributeError`` on every ``/teach status`` issued while recording. + """ recorder = getattr(self._pipeline, "recorder", None) - return recorder.step_count if recorder else 0 + trajectory = getattr(recorder, "current_trajectory", None) if recorder else None + return int(getattr(trajectory, "step_count", 0) or 0) @property def last_result(self) -> Optional["LearnResult"]: diff --git a/tests/.impact/coverage_map.json b/tests/.impact/coverage_map.json new file mode 100644 index 0000000..8470daa --- /dev/null +++ b/tests/.impact/coverage_map.json @@ -0,0 +1,756 @@ +{ + "_comment": "Generated by tools/impact.py --build-map. Maps each test file to the source files it actually executed, so change-scoped selection sees runtime coupling that a static import graph cannot.", + "tests": { + "tests/test_adaptive_depth.py": [ + "src/leapflow/config.py", + "src/leapflow/config_loader.py", + "src/leapflow/config_service.py", + "src/leapflow/domain/trajectory.py", + "src/leapflow/engine/agent_loop.py", + "src/leapflow/engine/budget.py", + "src/leapflow/engine/context_compressor.py", + "src/leapflow/engine/context_control.py", + "src/leapflow/engine/engine.py", + "src/leapflow/engine/prefix_commitment.py", + "src/leapflow/engine/research_ledger.py", + "src/leapflow/engine/subagent.py", + "src/leapflow/engine/turn_usage.py", + "src/leapflow/layout.py", + "src/leapflow/learning/difficulty_calibration.py", + "src/leapflow/llm/openai_provider.py", + "src/leapflow/security/path_sensitivity.py", + "src/leapflow/security/secrets.py", + "src/leapflow/storage/connection.py", + "src/leapflow/storage/duckdb_connect.py", + "src/leapflow/storage/evolution_store.py", + "src/leapflow/storage/research_ledger_store.py", + "src/leapflow/storage/write_buffer.py", + "src/leapflow/tools/registry_bootstrap.py" + ], + "tests/test_agent_execution.py": [ + "src/leapflow/config.py", + "src/leapflow/engine/context_compressor.py", + "src/leapflow/engine/context_control.py", + "src/leapflow/engine/context_disclosure.py", + "src/leapflow/engine/engine.py", + "src/leapflow/engine/error_classifier.py", + "src/leapflow/engine/execution_trace.py", + "src/leapflow/engine/failure_envelope.py", + "src/leapflow/engine/interaction_request.py", + "src/leapflow/engine/message_healer.py", + "src/leapflow/engine/oneshot_guard.py", + "src/leapflow/engine/prefix_commitment.py", + "src/leapflow/engine/recovery_audit.py", + "src/leapflow/engine/recovery_budget.py", + "src/leapflow/engine/recovery_checkpoint.py", + "src/leapflow/engine/recovery_coordinator.py", + "src/leapflow/engine/recovery_decision.py", + "src/leapflow/engine/recovery_strategies/__init__.py", + "src/leapflow/engine/recovery_strategies/context_compress.py", + "src/leapflow/engine/recovery_strategies/credential_rotate.py", + "src/leapflow/engine/recovery_strategies/jittered_retry.py", + "src/leapflow/engine/recovery_strategies/multimodal_strip.py", + "src/leapflow/engine/recovery_strategies/native_to_text.py", + "src/leapflow/engine/recovery_strategies/provider_failover.py", + "src/leapflow/engine/recovery_strategies/thinking_disable.py", + "src/leapflow/engine/recovery_strategies/tool_schema_expand.py", + "src/leapflow/engine/stale_stream.py", + "src/leapflow/engine/subagent.py", + "src/leapflow/engine/task_graph.py", + "src/leapflow/engine/tool_concurrency.py", + "src/leapflow/engine/tool_execution.py", + "src/leapflow/engine/tool_guardrails.py", + "src/leapflow/engine/turn_recovery.py", + "src/leapflow/engine/turn_usage.py", + "src/leapflow/engine/unified_classifier.py", + "src/leapflow/gateway/backends/cli_backend.py", + "src/leapflow/gateway/capability_health.py", + "src/leapflow/gateway/config_store.py", + "src/leapflow/gateway/connectors/action_registry.py", + "src/leapflow/gateway/connectors/cli_discovery.py", + "src/leapflow/gateway/credential_vault.py", + "src/leapflow/gateway/manifest.py", + "src/leapflow/gateway/resource_provenance.py", + "src/leapflow/gateway/server.py", + "src/leapflow/layout.py", + "src/leapflow/learning/active_learning.py", + "src/leapflow/learning/difficulty_calibration.py", + "src/leapflow/llm/message_builder.py", + "src/leapflow/memory/providers/episodic.py", + "src/leapflow/memory/providers/semantic.py", + "src/leapflow/memory/providers/working.py", + "src/leapflow/platform/mock.py", + "src/leapflow/security/permission_failures.py", + "src/leapflow/security/threat_patterns.py", + "src/leapflow/skills/registry.py", + "src/leapflow/skills/tool_executor.py", + "src/leapflow/tools/execution_context.py", + "src/leapflow/tools/file_operations.py", + "src/leapflow/tools/gateway_tool.py", + "src/leapflow/tools/name_resolver.py", + "src/leapflow/tools/registry_bootstrap.py", + "src/leapflow/tools/shell_tools.py", + "src/leapflow/tools/system_tools.py", + "src/leapflow/tools/text_tools.py", + "src/leapflow/world_model/orientation.py" + ], + "tests/test_app_connector.py": [ + "src/leapflow/gateway/adapters/common.py", + "src/leapflow/gateway/adapters/feishu.py", + "src/leapflow/gateway/backends/lark_cli_errors.py", + "src/leapflow/gateway/capability_health.py", + "src/leapflow/gateway/connectors/action_registry.py", + "src/leapflow/gateway/connectors/event_sources.py", + "src/leapflow/gateway/connectors/protocol.py", + "src/leapflow/gateway/resource_provenance.py", + "src/leapflow/gateway/server.py", + "src/leapflow/security/redact.py", + "src/leapflow/tools/gateway_tool.py" + ], + "tests/test_approval_layer.py": [ + "src/leapflow/cli/approval_view.py", + "src/leapflow/security/actions.py", + "src/leapflow/security/approval.py", + "src/leapflow/security/grants.py", + "src/leapflow/security/orchestrator.py", + "src/leapflow/security/path_sensitivity.py", + "src/leapflow/security/policy.py", + "src/leapflow/security/redact.py", + "src/leapflow/security/risk.py", + "src/leapflow/security/threat_patterns.py", + "src/leapflow/tools/file_operations.py", + "src/leapflow/tools/registry_bootstrap.py" + ], + "tests/test_architecture_contracts.py": [ + "src/leapflow/daemon/notifications.py", + "src/leapflow/engine/session_factory.py", + "src/leapflow/logging_setup.py" + ], + "tests/test_board_session_binding.py": [ + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/daemon/session_coordinator.py", + "src/leapflow/daemon/session_registry.py", + "src/leapflow/monitor/series_extractor.py", + "src/leapflow/monitor/session_producer.py", + "src/leapflow/monitor/types.py" + ], + "tests/test_budget_calibration.py": [ + "src/leapflow/engine/context_control.py", + "src/leapflow/engine/engine.py" + ], + "tests/test_cache_manager.py": [ + "src/leapflow/cache/manager.py", + "src/leapflow/layout.py" + ], + "tests/test_cli_discovery.py": [ + "src/leapflow/gateway/connectors/action_registry.py", + "src/leapflow/gateway/connectors/cli_discovery.py" + ], + "tests/test_cli_entrypoint.py": [ + "src/leapflow/analysis/abstractor.py", + "src/leapflow/analysis/consensus.py", + "src/leapflow/analysis/denoise.py", + "src/leapflow/analysis/fs_pattern_pass.py", + "src/leapflow/analysis/intent_inferrer.py", + "src/leapflow/analysis/patterns.py", + "src/leapflow/analysis/pipeline.py", + "src/leapflow/analysis/segmenter.py", + "src/leapflow/analysis/synthesis.py", + "src/leapflow/causal/channel.py", + "src/leapflow/causal/components.py", + "src/leapflow/causal/inference.py", + "src/leapflow/causal/pipeline.py", + "src/leapflow/causal/types.py", + "src/leapflow/cli/cli.py", + "src/leapflow/cli/commands/config.py", + "src/leapflow/cli/commands/daemon.py", + "src/leapflow/cli/commands/host.py", + "src/leapflow/cli/commands/interactive.py", + "src/leapflow/cli/commands/registry.py", + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/cli/context.py", + "src/leapflow/cli/helpers.py", + "src/leapflow/config.py", + "src/leapflow/config_loader.py", + "src/leapflow/config_service.py", + "src/leapflow/copilot/__init__.py", + "src/leapflow/copilot/adapters.py", + "src/leapflow/copilot/config.py", + "src/leapflow/copilot/context.py", + "src/leapflow/copilot/degradation.py", + "src/leapflow/copilot/engine.py", + "src/leapflow/copilot/feedback.py", + "src/leapflow/copilot/idle.py", + "src/leapflow/copilot/pipeline.py", + "src/leapflow/copilot/predictors/__init__.py", + "src/leapflow/copilot/predictors/l0_hash.py", + "src/leapflow/copilot/predictors/l1_markov.py", + "src/leapflow/copilot/predictors/l2_embed.py", + "src/leapflow/copilot/predictors/l3_llm.py", + "src/leapflow/copilot/renderer.py", + "src/leapflow/copilot/types.py", + "src/leapflow/domain/platform.py", + "src/leapflow/domain/trajectory.py", + "src/leapflow/domain/ui_vocabulary.py", + "src/leapflow/engine/audit.py", + "src/leapflow/engine/confirmation.py", + "src/leapflow/engine/context_compressor.py", + "src/leapflow/engine/engine.py", + "src/leapflow/engine/graph_planner.py", + "src/leapflow/engine/intent_classifier.py", + "src/leapflow/engine/pipeline_observer.py", + "src/leapflow/engine/prompt_cache.py", + "src/leapflow/engine/scheduler.py", + "src/leapflow/engine/session.py", + "src/leapflow/engine/situational_assessor.py", + "src/leapflow/engine/tool_guardrails.py", + "src/leapflow/gateway/checkpoint_store.py", + "src/leapflow/gateway/config_store.py", + "src/leapflow/gateway/event_bridge.py", + "src/leapflow/gateway/manifest.py", + "src/leapflow/gateway/normalizers/dingtalk.py", + "src/leapflow/gateway/normalizers/feishu.py", + "src/leapflow/gateway/normalizers/telegram.py", + "src/leapflow/gateway/router.py", + "src/leapflow/gateway/server.py", + "src/leapflow/gateway/trigger_policy.py", + "src/leapflow/layout.py", + "src/leapflow/learning/active_learning.py", + "src/leapflow/learning/codegen.py", + "src/leapflow/learning/cold_start.py", + "src/leapflow/learning/distiller.py", + "src/leapflow/learning/doc_generator.py", + "src/leapflow/learning/effectiveness.py", + "src/leapflow/learning/feedback.py", + "src/leapflow/learning/learnability.py", + "src/leapflow/learning/similarity.py", + "src/leapflow/llm/model_capabilities.py", + "src/leapflow/llm/openai_provider.py", + "src/leapflow/llm/provider_chain.py", + "src/leapflow/logging_setup.py", + "src/leapflow/memory/manager.py", + "src/leapflow/memory/providers/episodic.py", + "src/leapflow/memory/providers/evolution.py", + "src/leapflow/memory/providers/narrative.py", + "src/leapflow/memory/providers/semantic.py", + "src/leapflow/memory/providers/working.py", + "src/leapflow/perception/config.py", + "src/leapflow/perception/session.py", + "src/leapflow/perception/signals.py", + "src/leapflow/perception/state_snapshot.py", + "src/leapflow/perception/storage/frame_store.py", + "src/leapflow/platform/adapters/mock.py", + "src/leapflow/platform/cua_client.py", + "src/leapflow/platform/event_bus.py", + "src/leapflow/platform/facade.py", + "src/leapflow/platform/mock.py", + "src/leapflow/platform/normalizer.py", + "src/leapflow/privacy/__init__.py", + "src/leapflow/privacy/policy.py", + "src/leapflow/recording/attention.py", + "src/leapflow/recording/attention_tuner.py", + "src/leapflow/recording/recorder.py", + "src/leapflow/security/approval.py", + "src/leapflow/security/orchestrator.py", + "src/leapflow/security/redact.py", + "src/leapflow/security/secrets.py", + "src/leapflow/skills/activator.py", + "src/leapflow/skills/bridge_factory.py", + "src/leapflow/skills/discovery.py", + "src/leapflow/skills/evolution.py", + "src/leapflow/skills/index.py", + "src/leapflow/skills/injector.py", + "src/leapflow/skills/registry.py", + "src/leapflow/skills/semantic_adapter.py", + "src/leapflow/skills/tool_executor.py", + "src/leapflow/skills/ui_selector.py", + "src/leapflow/skills/ui_summarizer.py", + "src/leapflow/storage/connection.py", + "src/leapflow/storage/conversation_store.py", + "src/leapflow/storage/duckdb_connect.py", + "src/leapflow/storage/reentry_store.py", + "src/leapflow/storage/session_store.py", + "src/leapflow/storage/skill_docs.py", + "src/leapflow/storage/skill_library.py", + "src/leapflow/storage/trajectory_store.py", + "src/leapflow/storage/write_buffer.py", + "src/leapflow/tools/config_tools.py", + "src/leapflow/tools/dev_tools.py", + "src/leapflow/tools/file_operations.py", + "src/leapflow/tools/registry_bootstrap.py", + "src/leapflow/tools/shell_tools.py", + "src/leapflow/tools/terminal_session.py", + "src/leapflow/tools/web_fetch.py", + "src/leapflow/world_model/budget.py", + "src/leapflow/world_model/curiosity.py", + "src/leapflow/world_model/embedding.py", + "src/leapflow/world_model/experience_store.py", + "src/leapflow/world_model/prediction.py", + "src/leapflow/world_model/replay.py", + "src/leapflow/world_model/trajectory_grader.py" + ], + "tests/test_cli_ndjson_event_source.py": [ + "src/leapflow/gateway/connectors/event_sources.py" + ], + "tests/test_code_tools.py": [ + "src/leapflow/cli/context.py", + "src/leapflow/security/actions.py", + "src/leapflow/security/orchestrator.py", + "src/leapflow/security/policy.py", + "src/leapflow/security/redact.py", + "src/leapflow/security/risk.py", + "src/leapflow/tools/code_intel.py", + "src/leapflow/tools/file_operations.py" + ], + "tests/test_config_and_path_contracts.py": [ + "src/leapflow/config_service.py", + "src/leapflow/dashboard/intent.py", + "src/leapflow/dashboard/service.py", + "src/leapflow/dashboard/templates.py", + "src/leapflow/dashboard/viewspec.py", + "src/leapflow/layout.py" + ], + "tests/test_config_capability_tools.py": [ + "src/leapflow/config_service.py", + "src/leapflow/daemon/_service_helpers.py", + "src/leapflow/layout.py", + "src/leapflow/security/orchestrator.py", + "src/leapflow/security/risk.py", + "src/leapflow/tools/config_tools.py", + "src/leapflow/tools/execution_context.py" + ], + "tests/test_config_loader.py": [ + "src/leapflow/cli/commands/daemon.py", + "src/leapflow/config_loader.py", + "src/leapflow/logging_setup.py", + "src/leapflow/security/secrets.py" + ], + "tests/test_context_budget_scaling.py": [ + "src/leapflow/engine/context_compressor.py", + "src/leapflow/engine/engine.py", + "src/leapflow/llm/model_capabilities.py" + ], + "tests/test_context_disclosure.py": [ + "src/leapflow/engine/context_disclosure.py" + ], + "tests/test_context_governance.py": [ + "src/leapflow/engine/context_compressor.py", + "src/leapflow/engine/context_control.py", + "src/leapflow/tools/file_operations.py" + ], + "tests/test_daemon_event_loop_blocking.py": [ + "src/leapflow/cli/context.py", + "src/leapflow/daemon/_service_helpers.py", + "src/leapflow/daemon/approval_coordinator.py", + "src/leapflow/daemon/monitor_coordinator.py", + "src/leapflow/daemon/notifications.py", + "src/leapflow/daemon/reentry_coordinator.py", + "src/leapflow/daemon/service.py", + "src/leapflow/daemon/turn_admission.py", + "src/leapflow/layout.py", + "src/leapflow/memory/providers/semantic.py" + ], + "tests/test_daemon_isolation.py": [ + "src/leapflow/daemon/approval_coordinator.py", + "src/leapflow/daemon/service.py", + "src/leapflow/memory/manager.py", + "src/leapflow/memory/protocol.py", + "src/leapflow/memory/providers/episodic.py", + "src/leapflow/memory/providers/semantic.py" + ], + "tests/test_daemon_rpc.py": [ + "src/leapflow/cli/banner.py", + "src/leapflow/cli/commands/daemon.py", + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/daemon/_service_helpers.py", + "src/leapflow/daemon/approval_coordinator.py", + "src/leapflow/daemon/client.py", + "src/leapflow/daemon/lease.py", + "src/leapflow/daemon/lifecycle.py", + "src/leapflow/daemon/monitor_coordinator.py", + "src/leapflow/daemon/protocol.py", + "src/leapflow/daemon/reentry_coordinator.py", + "src/leapflow/daemon/server.py", + "src/leapflow/daemon/service.py", + "src/leapflow/daemon/session_coordinator.py", + "src/leapflow/daemon/session_registry.py", + "src/leapflow/daemon/turn_admission.py", + "src/leapflow/engine/engine.py", + "src/leapflow/engine/session_factory.py", + "src/leapflow/gateway/server.py", + "src/leapflow/monitor/finding_store.py", + "src/leapflow/monitor/manager.py", + "src/leapflow/monitor/producers.py", + "src/leapflow/monitor/session_producer.py", + "src/leapflow/platform/mock.py", + "src/leapflow/scheduler/coordinator.py", + "src/leapflow/scheduler/local_scheduler.py", + "src/leapflow/scheduler/store.py", + "src/leapflow/security/orchestrator.py", + "src/leapflow/tools/gateway_tool.py" + ], + "tests/test_dashboard_domains.py": [ + "src/leapflow/dashboard/templates.py" + ], + "tests/test_dashboard_launcher.py": [ + "src/leapflow/dashboard/hub.py", + "src/leapflow/dashboard/launcher.py", + "src/leapflow/dashboard/server.py", + "src/leapflow/dashboard/service.py" + ], + "tests/test_dashboard_sdui.py": [ + "src/leapflow/dashboard/intent.py", + "src/leapflow/dashboard/templates.py", + "src/leapflow/dashboard/viewspec.py" + ], + "tests/test_dashboard_view.py": [ + "src/leapflow/dashboard/hub.py", + "src/leapflow/dashboard/service.py", + "src/leapflow/dashboard/templates.py" + ], + "tests/test_dashboard_watch_rpc.py": [ + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/daemon/monitor_coordinator.py", + "src/leapflow/daemon/service.py", + "src/leapflow/dashboard/launcher.py", + "src/leapflow/dashboard/templates.py", + "src/leapflow/layout.py", + "src/leapflow/monitor/finding_store.py", + "src/leapflow/monitor/manager.py", + "src/leapflow/monitor/producers.py", + "src/leapflow/monitor/types.py", + "src/leapflow/scheduler/coordinator.py", + "src/leapflow/scheduler/local_scheduler.py", + "src/leapflow/scheduler/store.py", + "src/leapflow/scheduler/triggers/__init__.py", + "src/leapflow/scheduler/triggers/interval.py", + "src/leapflow/scheduler/types.py" + ], + "tests/test_deferred_init_responsiveness.py": [ + "src/leapflow/cli/context.py" + ], + "tests/test_dev_terminal_tools.py": [ + "src/leapflow/tools/dev_tools.py", + "src/leapflow/tools/shell_tools.py", + "src/leapflow/tools/terminal_session.py" + ], + "tests/test_empty_response_hardening.py": [ + "src/leapflow/engine/engine.py" + ], + "tests/test_execution_backends.py": [ + "src/leapflow/gateway/backends/cli_backend.py", + "src/leapflow/gateway/backends/lark_cli_errors.py", + "src/leapflow/gateway/backends/rest_backend.py" + ], + "tests/test_feishu_event_normalizer.py": [ + "src/leapflow/gateway/normalizers/feishu.py" + ], + "tests/test_gateway_adapters.py": [ + "src/leapflow/gateway/adapters/api_server.py", + "src/leapflow/gateway/adapters/common.py", + "src/leapflow/gateway/adapters/dingtalk.py", + "src/leapflow/gateway/adapters/feishu.py", + "src/leapflow/gateway/adapters/telegram.py", + "src/leapflow/gateway/adapters/webhook.py", + "src/leapflow/gateway/connectors/dingtalk_event_source.py", + "src/leapflow/gateway/connectors/telegram_event_source.py", + "src/leapflow/gateway/server.py" + ], + "tests/test_gateway_consumer_loop.py": [ + "src/leapflow/gateway/server.py", + "src/leapflow/gateway/session_router.py", + "src/leapflow/gateway/trigger_policy.py" + ], + "tests/test_gateway_tool_e2e.py": [ + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/gateway/adapters/feishu.py", + "src/leapflow/gateway/backends/cli_backend.py", + "src/leapflow/gateway/capability_health.py", + "src/leapflow/gateway/checkpoint_store.py", + "src/leapflow/gateway/config_store.py", + "src/leapflow/gateway/connectors/composite_event_source.py", + "src/leapflow/gateway/connectors/event_sources.py", + "src/leapflow/gateway/connectors/lark_event_source.py", + "src/leapflow/gateway/credential_vault.py", + "src/leapflow/gateway/server.py", + "src/leapflow/security/redact.py", + "src/leapflow/tools/gateway_tool.py" + ], + "tests/test_internal_defect_reporting.py": [ + "src/leapflow/engine/engine.py", + "src/leapflow/engine/error_classifier.py", + "src/leapflow/engine/recovery_coordinator.py", + "src/leapflow/engine/unified_classifier.py" + ], + "tests/test_journey_harness.py": [ + "src/leapflow/llm/openai_provider.py" + ], + "tests/test_layout.py": [ + "src/leapflow/layout.py" + ], + "tests/test_memory_and_storage.py": [ + "src/leapflow/domain/trajectory.py", + "src/leapflow/memory/manager.py", + "src/leapflow/memory/providers/episodic.py", + "src/leapflow/memory/providers/semantic.py", + "src/leapflow/memory/providers/working.py", + "src/leapflow/platform/event_bus.py", + "src/leapflow/platform/reorder_buffer.py", + "src/leapflow/storage/connection.py", + "src/leapflow/storage/conversation_store.py", + "src/leapflow/storage/duckdb_connect.py", + "src/leapflow/storage/skill_library.py", + "src/leapflow/storage/trajectory_store.py", + "src/leapflow/storage/write_buffer.py" + ], + "tests/test_monitor_subsystem.py": [ + "src/leapflow/monitor/finding_store.py", + "src/leapflow/monitor/manager.py", + "src/leapflow/monitor/types.py", + "src/leapflow/scheduler/store.py" + ], + "tests/test_orientation.py": [ + "src/leapflow/world_model/orientation.py" + ], + "tests/test_path_sensitivity.py": [ + "src/leapflow/layout.py", + "src/leapflow/security/path_sensitivity.py", + "src/leapflow/security/risk.py" + ], + "tests/test_perception_pipeline.py": [ + "src/leapflow/causal/channel.py", + "src/leapflow/causal/components.py", + "src/leapflow/causal/inference.py", + "src/leapflow/causal/types.py", + "src/leapflow/signal_fusion/action_agent.py", + "src/leapflow/signal_fusion/episode_agent.py", + "src/leapflow/signal_fusion/integrator.py", + "src/leapflow/signal_fusion/pipeline.py", + "src/leapflow/signal_fusion/quality.py", + "src/leapflow/signal_fusion/segment_agent.py", + "src/leapflow/signal_fusion/types.py", + "src/leapflow/signal_fusion/wait_classifier.py", + "src/leapflow/utils/diagnostics.py" + ], + "tests/test_platform_synthesis.py": [ + "src/leapflow/analysis/denoise.py", + "src/leapflow/analysis/synthesis.py" + ], + "tests/test_pure_algorithms.py": [ + "src/leapflow/causal/types.py", + "src/leapflow/learning/active_learning.py", + "src/leapflow/learning/similarity.py", + "src/leapflow/memory/__init__.py", + "src/leapflow/world_model/_json_utils.py" + ], + "tests/test_recovery_audit.py": [ + "src/leapflow/engine/recovery_audit.py", + "src/leapflow/engine/recovery_budget.py" + ], + "tests/test_recovery_checkpoint.py": [ + "src/leapflow/engine/recovery_checkpoint.py" + ], + "tests/test_recovery_contract_e2e.py": [ + "src/leapflow/engine/oneshot_guard.py", + "src/leapflow/engine/recovery_budget.py", + "src/leapflow/engine/recovery_coordinator.py", + "src/leapflow/engine/recovery_decision.py", + "src/leapflow/engine/recovery_strategies/context_compress.py", + "src/leapflow/engine/recovery_strategies/credential_rotate.py", + "src/leapflow/engine/recovery_strategies/jittered_retry.py", + "src/leapflow/engine/recovery_strategies/multimodal_strip.py", + "src/leapflow/engine/recovery_strategies/native_to_text.py", + "src/leapflow/engine/recovery_strategies/provider_failover.py", + "src/leapflow/engine/recovery_strategies/thinking_disable.py", + "src/leapflow/engine/recovery_strategies/tool_schema_expand.py", + "src/leapflow/engine/unified_classifier.py" + ], + "tests/test_recovery_coordinator.py": [ + "src/leapflow/engine/failure_envelope.py", + "src/leapflow/engine/oneshot_guard.py", + "src/leapflow/engine/recovery_budget.py", + "src/leapflow/engine/recovery_coordinator.py", + "src/leapflow/engine/recovery_decision.py" + ], + "tests/test_recovery_strategies.py": [ + "src/leapflow/engine/recovery_strategies/multimodal_strip.py", + "src/leapflow/engine/recovery_strategies/native_to_text.py", + "src/leapflow/engine/recovery_strategies/thinking_disable.py", + "src/leapflow/engine/recovery_strategies/tool_schema_expand.py" + ], + "tests/test_reentry_driver.py": [ + "src/leapflow/scheduler/reentry_driver.py", + "src/leapflow/storage/reentry_store.py" + ], + "tests/test_reentry_send.py": [ + "src/leapflow/scheduler/reentry_send.py", + "src/leapflow/security/send_trust.py" + ], + "tests/test_reentry_service.py": [ + "src/leapflow/scheduler/reentry_driver.py", + "src/leapflow/scheduler/reentry_send.py", + "src/leapflow/scheduler/reentry_service.py", + "src/leapflow/storage/reentry_store.py" + ], + "tests/test_reentry_store.py": [ + "src/leapflow/storage/reentry_store.py", + "src/leapflow/tools/registry_bootstrap.py" + ], + "tests/test_repo_map.py": [ + "src/leapflow/tools/dev_tools.py", + "src/leapflow/tools/repo_map.py" + ], + "tests/test_runtime_metadata_and_wrapping.py": [ + "src/leapflow/cli/tui_app/console.py", + "src/leapflow/cli/tui_app/theme.py", + "src/leapflow/daemon/service.py" + ], + "tests/test_safety_and_policy.py": [ + "src/leapflow/domain/skill_types.py", + "src/leapflow/engine/confirmation.py", + "src/leapflow/gateway/server.py", + "src/leapflow/gateway/session_router.py", + "src/leapflow/skills/action_policy.py", + "src/leapflow/skills/sandbox.py", + "src/leapflow/tools/file_operations.py" + ], + "tests/test_scm_tools.py": [ + "src/leapflow/tools/scm_tools.py" + ], + "tests/test_series_extractor.py": [ + "src/leapflow/monitor/series_extractor.py" + ], + "tests/test_session_analysis.py": [ + "src/leapflow/daemon/service.py", + "src/leapflow/daemon/session_coordinator.py", + "src/leapflow/monitor/series_extractor.py", + "src/leapflow/monitor/session_producer.py" + ], + "tests/test_session_factory.py": [ + "src/leapflow/engine/engine.py", + "src/leapflow/engine/tool_concurrency.py" + ], + "tests/test_session_registry.py": [ + "src/leapflow/daemon/session_registry.py" + ], + "tests/test_skill_lifecycle.py": [ + "src/leapflow/learning/doc_generator.py", + "src/leapflow/learning/document.py", + "src/leapflow/platform/adapters/darwin.py", + "src/leapflow/skills/registry.py", + "src/leapflow/storage/skill_docs.py", + "src/leapflow/utils/resilience.py" + ], + "tests/test_slash_command_router.py": [ + "src/leapflow/cli/commands/interactive.py", + "src/leapflow/cli/commands/registry.py", + "src/leapflow/cli/commands/router.py", + "src/leapflow/cli/commands/slash_handlers.py", + "src/leapflow/world_model/orientation.py" + ], + "tests/test_teach_learn_lifecycle.py": [ + "src/leapflow/analysis/abstractor.py", + "src/leapflow/analysis/causal.py", + "src/leapflow/analysis/denoise.py", + "src/leapflow/analysis/episode_dedup.py", + "src/leapflow/analysis/fs_pattern_pass.py", + "src/leapflow/analysis/patterns.py", + "src/leapflow/analysis/pipeline.py", + "src/leapflow/analysis/segmenter.py", + "src/leapflow/analysis/synthesis.py", + "src/leapflow/domain/trajectory.py", + "src/leapflow/engine/session.py", + "src/leapflow/learning/active_learning.py", + "src/leapflow/learning/distiller.py", + "src/leapflow/learning/feedback.py", + "src/leapflow/learning/similarity.py", + "src/leapflow/memory/providers/working.py", + "src/leapflow/recording/attention.py", + "src/leapflow/recording/recorder.py", + "src/leapflow/storage/skill_library.py", + "src/leapflow/storage/trajectory_store.py" + ], + "tests/test_tool_call_hardening.py": [ + "src/leapflow/engine/engine.py", + "src/leapflow/tools/file_operations.py", + "src/leapflow/tools/shell_tools.py" + ], + "tests/test_tool_concurrency.py": [ + "src/leapflow/engine/tool_concurrency.py", + "src/leapflow/engine/tool_execution.py" + ], + "tests/test_trigger_policy.py": [ + "src/leapflow/gateway/trigger_policy.py" + ], + "tests/test_tui_command_queue.py": [ + "src/leapflow/cli/approval_view.py", + "src/leapflow/cli/commands/interactive.py", + "src/leapflow/cli/tui_app/app.py", + "src/leapflow/cli/tui_app/approval_modal.py", + "src/leapflow/cli/tui_app/command.py", + "src/leapflow/cli/tui_app/console.py", + "src/leapflow/cli/tui_app/input.py", + "src/leapflow/cli/tui_app/paste.py", + "src/leapflow/cli/tui_app/stream.py" + ], + "tests/test_tui_session_summary.py": [ + "src/leapflow/cli/cli.py", + "src/leapflow/cli/tui_app/session_summary.py", + "src/leapflow/cli/tui_app/stream.py" + ], + "tests/test_tui_theme.py": [ + "src/leapflow/cli/banner.py", + "src/leapflow/cli/tui_app/console.py", + "src/leapflow/cli/tui_app/status.py", + "src/leapflow/cli/tui_app/theme.py" + ], + "tests/test_tui_tool_audit.py": [ + "src/leapflow/cli/tui_app/stream.py", + "src/leapflow/engine/engine.py", + "src/leapflow/engine/tool_execution.py" + ], + "tests/test_turn_admission.py": [ + "src/leapflow/daemon/turn_admission.py" + ], + "tests/test_uncertain_effect_and_interaction.py": [ + "src/leapflow/engine/engine.py", + "src/leapflow/engine/interaction_request.py", + "src/leapflow/engine/tool_execution.py" + ], + "tests/test_unified_classifier.py": [ + "src/leapflow/engine/error_classifier.py", + "src/leapflow/engine/unified_classifier.py" + ], + "tests/test_visual_pipeline.py": [ + "src/leapflow/analysis/abstractor.py", + "src/leapflow/analysis/pipeline.py", + "src/leapflow/domain/trajectory.py", + "src/leapflow/perception/video/analyzer.py", + "src/leapflow/perception/video/prompts.py", + "src/leapflow/perception/video/recorder.py", + "src/leapflow/perception/video/segmenter.py", + "src/leapflow/perception/video/timeline.py", + "src/leapflow/recording/attention.py", + "src/leapflow/storage/trajectory_store.py" + ], + "tests/test_web_fetch.py": [ + "src/leapflow/cache/manager.py", + "src/leapflow/engine/context_control.py", + "src/leapflow/layout.py", + "src/leapflow/security/actions.py", + "src/leapflow/security/network.py", + "src/leapflow/security/risk.py", + "src/leapflow/tools/web_cache.py", + "src/leapflow/tools/web_extract.py", + "src/leapflow/tools/web_fetch.py" + ], + "tests/test_world_model.py": [ + "src/leapflow/memory/providers/semantic.py", + "src/leapflow/world_model/budget.py", + "src/leapflow/world_model/curiosity.py", + "src/leapflow/world_model/experience_store.py", + "src/leapflow/world_model/replay.py", + "src/leapflow/world_model/trajectory_grader.py" + ] + } +} diff --git a/tests/.impact/escalate.yaml b/tests/.impact/escalate.yaml new file mode 100644 index 0000000..73b8115 --- /dev/null +++ b/tests/.impact/escalate.yaml @@ -0,0 +1,48 @@ +# Changes that force the whole mock layer to run. +# +# Change-scoped selection is a cost optimisation, and it must never be the reason +# a regression escapes. Anything listed here is a shared foundation: its blast +# radius is either the entire codebase or impossible to bound statically, so the +# honest answer is to run everything. +# +# Patterns are matched with pathlib's `PurePath.match` against repo-relative +# paths, so `src/leapflow/domain/*.py` matches one level and `**/x.py` matches any +# depth. Read by tools/impact.py. + +patterns: + # Configuration and path layout: read by essentially every module, and their + # precedence rules are the kind of thing that breaks distant code silently. + - src/leapflow/config.py + - src/leapflow/config_loader.py + - src/leapflow/config_service.py + - src/leapflow/layout.py + + # Domain types travel everywhere by value; a field change is unbounded. + - src/leapflow/domain/*.py + + # The agent loop and its recovery pipeline: coupled to tools, memory, LLM and + # the daemon through event and protocol indirection that no import graph shows. + - src/leapflow/engine/engine.py + - src/leapflow/engine/agent_loop.py + - src/leapflow/engine/session_factory.py + - src/leapflow/engine/recovery_coordinator.py + - src/leapflow/engine/unified_classifier.py + + # Daemon service and session routing: every client-visible value flows through + # here, and multi-client leakage is invisible from any single module's tests. + - src/leapflow/daemon/service.py + - src/leapflow/daemon/session_registry.py + - src/leapflow/daemon/session_coordinator.py + - src/leapflow/daemon/protocol.py + + # Test infrastructure: changing it changes the meaning of every result. + - tests/conftest.py + - tests/_harness/*.py + + # Build, dependency and lane definitions. + - pyproject.toml + - uv.lock + - Makefile + - .github/workflows/*.yaml + - tools/impact.py + - tests/.impact/escalate.yaml diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..4b4d28b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,289 @@ +# LeapFlow Test Suite + +## Two-Layer Architecture + +**Mock layer** (`tests/*.py`) — Fast, hermetic unit/component tests. No network, no LLM calls. Uses `StubLLM` for deterministic responses. +Markers: `unit`, `component`. + +**Real layer** (`tests/journeys/`) — 6 coarse-grained end-to-end journeys driving a real `leapd` subprocess over RPC with the LLM boundary served by a local cassette proxy. +Markers: `e2e`, `slow`. + +--- + +## CI Pipeline (3 Tiers) + +| Tier | Workflow | Trigger | What runs | +|------|----------|---------|-----------| +| **PR Gate** | `ci.yaml` → `pr` job | Pull request | Lint → fixture consistency → real layer (journeys + regression) → **full** mock layer | +| **Main Full** | `ci.yaml` → `main` job | Push to main | Matrix (ubuntu+macos × Python 3.11/3.12/3.13), full mock + real layer | +| **Nightly Live** | `nightly-live.yaml` → `live` job | Cron, `workflow_dispatch`, or a PR labelled `ci:live` | Live-capable journeys against a real provider | +| **Re-record** | `nightly-live.yaml` → `rerecord` job | `workflow_dispatch` only | Captures real traffic into `recordings/`, opens a PR | +| **Impact map** | `nightly-live.yaml` → `impact-map` job | Cron / dispatch | Rebuilds `tests/.impact/coverage_map.json`, opens a PR | + +Neither offline lane selects tests. The always-on tiers cannot be selected away +and already dominate the run (~14s of ~18s), so scoping the mock layer would save +only a few seconds — not worth any risk of under-selecting. The **live** lane does +select, because there each journey costs real tokens. + +Only the two live-provider jobs need credentials — see [Credentials](#credentials). +The PR and main gates are fully offline by design. + +--- + +## Make Targets + +| Target | Purpose | +|--------|---------| +| `make test` | Default gate: mock + e2e | +| `make test-unit` | Mock layer only (`-m "not e2e"`) | +| `make test-e2e` | Real layer (journeys + regression) | +| `make test-full` | All tests, no filter | +| `make test-impact` | Change-scoped mock layer (local convenience; not used by CI) | +| `make test-live` | Real provider (requires credentials) | +| `make seed-cassettes` | Rebuild offline replay store | +| `make record-traffic` | Record real provider traffic | +| `make sync-fixtures` | Derive mock-layer response shapes from recordings | + +--- + +## Harness (`tests/_harness/`) + +| Module | Role | +|--------|------| +| `cassette_proxy.py` | Local OpenAI-compatible HTTP endpoint with 4 modes: `replay`, `seed`, `record`, `live` | +| `cassette.py` | Fingerprint computation, cassette persistence, miss diagnostics | +| `leapd.py` | Spawn real daemon subprocess with environment isolation | +| `journey.py` | Journey runner with phase attribution, deadline, call-budget and token-budget enforcement | + +--- + +## Fixtures (`tests/_fixtures/`) + +- `cassettes/` — Per-journey deterministic replay data (rebuilt by `make seed-cassettes`) +- `recordings/` — Real provider traffic evidence (captured by `make record-traffic`) +- `llm_responses/response_shapes.json` — Derived fixture asserted by mock layer (rebuilt by `make sync-fixtures`) + +--- + +## Regression Guards (`tests/regression/`) + +| File | Purpose | +|------|---------| +| `test_suite_budget.py` | Hard ceiling on journey count (merge, don't raise) | +| `test_incident_ledger.py` | Ensures past-escaped incidents stay covered | +| `test_test_layer_contracts.py` | Meta-tests preventing suite degradation | +| `test_provider_shape_drift.py` | Catches provider response format changes | +| `test_impact_selection.py` | Validates change-scope selection logic | + +--- + +## Cost and Convergence Guards + +Every journey declares two ceilings, both enforced at the cassette proxy and +reported by `journey.finish()`. Exceeding either returns HTTP 400 — non-retryable +on purpose, so a runaway loop stops at the ceiling instead of feeding the +provider's retry logic. + +| Ceiling | Default | Catches | +|---------|---------|---------| +| `max_llm_calls` | `DEFAULT_MAX_LLM_CALLS = 12` | A turn that stops converging and keeps re-asking the model | +| `max_llm_tokens` | `DEFAULT_MAX_LLM_TOKENS = 150_000` | Prompt growth — a longer system prompt or bigger tool catalogue raises cost *without* adding a round, which call count cannot see | + +Measured against a real provider (qwen3.7-plus), the same journey has cost between +4 and 7 provider calls on different runs: the model decides how many tool +round-trips to make. The ceilings are sized to absorb that swing, so raising one +means investigating, not editing. + +--- + +## Live Journey Selection + +Each journey declares its own metadata, read by `tools/impact.py` via AST (never +imported): + +- `SUBJECT_PATHS` — the source areas the journey exercises +- `LIVE_SIGNAL` — whether running it against a real provider adds signal + +`tools/impact.py --live-journeys` picks from those. A scheduled run takes every +live-capable journey; a `ci:live`-labelled PR takes only the journeys whose +subjects the change touches. `LIVE_SIGNAL = False` journeys (control plane, +lifecycle) never run live. `test_r4_recovery.py` additionally refuses to run live +at runtime, because every response it asserts on is an injected failure that a +forwarding mode cannot produce. + +--- + +## LLM Test Modes + +Controlled by `LEAPFLOW_TEST_LLM_MODE` env var: + +| Mode | Reaches a provider? | Persists to | Behaviour | +|------|--------------------|-------------|-----------| +| `replay` | No | — | CI default. Deterministic playback from `cassettes/`; a miss fails with a nearest-neighbour diff | +| `seed` | **No** | `cassettes/` | Serves the journey's declared script and stores it, building the offline replay store without any credential | +| `record` | **Yes** | `recordings/` | Forwards to the real provider and stores the response as wire-shape evidence — never into the replay store | +| `live` | **Yes** | nothing | Runs against the real provider, persists nothing | + +A multi-turn agent conversation **cannot** be replayed from a recording: turn *n*'s +prompt embeds the exact round-by-round history of turns 1..n-1, so one divergence +(a tool call the model made this time but not last time) cascades. That is why +`record` writes to a separate store and can never break the offline lanes. + +--- + +## Credentials + +### The offline lanes need none + +`ci.yaml` (`pr` and `main`) sets `LEAPFLOW_TEST_LLM_MODE: replay` and serves the +LLM boundary from the committed cassette store. This is a requirement, not a +convenience: a pull request from a fork cannot read secrets, so anything that +gates a merge has to run without them. Never add a secret dependency to those +jobs — a credential in the PR gate silently stops fork contributions from being +mergeable. + +### The live lanes need exactly three + +Only `nightly-live.yaml` reaches a real provider. Both of its credential-using +jobs (`live`, `rerecord`) declare `environment: live-llm`, so configure these as +**environment** secrets under that environment — not repository secrets. + +Note that a secret's name and the environment variable it feeds are two separate +namespaces, and for the model they deliberately differ: + +```yaml +# nightly-live.yaml +LEAPFLOW_LLM_API_KEY: ${{ secrets.LEAPFLOW_LLM_API_KEY }} +LEAPFLOW_LLM_BASE_URL: ${{ secrets.LEAPFLOW_LLM_BASE_URL }} +LEAPFLOW_LLM_MODEL: ${{ secrets.LEAPFLOW_LLM_CHEAP_MODEL }} +# ↑ env var the test process reads ↑ secret name you create +``` + +| Create this secret | It is injected as | Example | +|--------------------|-------------------|---------| +| `LEAPFLOW_LLM_API_KEY` | `LEAPFLOW_LLM_API_KEY` | provider API key | +| `LEAPFLOW_LLM_BASE_URL` | `LEAPFLOW_LLM_BASE_URL` | `https://dashscope.aliyuncs.com/compatible-mode/v1` | +| **`LEAPFLOW_LLM_CHEAP_MODEL`** | **`LEAPFLOW_LLM_MODEL`** | `qwen3.7-plus` | + +The model secret carries `CHEAP` in its name on purpose: journeys assert +invariants, not prose quality, so the lane should run on the cheapest model that +still follows tool-calling instructions. The name is the reminder. + +> **Do not create a secret literally named `LEAPFLOW_LLM_MODEL`.** Nothing reads +> it — `nightly-live.yaml` only ever references `secrets.LEAPFLOW_LLM_CHEAP_MODEL`. +> A secret under the wrong name leaves the model empty, which skips rather than +> fails (see below). + +Resolution path: the workflow injects `LEAPFLOW_LLM_*` → `upstream_from_env()` +reads `LEAPFLOW_TEST_UPSTREAM_*` first and falls back to `LEAPFLOW_LLM_*` → the +`journey_mode` fixture **skips** (not fails) when any of the three is empty. A +live lane that reports "skipped" is a missing or misnamed credential, not a +passing run — check the job output shows journeys actually executing. + +Two details worth getting right when setting this up: + +- **Create the `live-llm` environment before the first run.** If it does not + exist, GitHub auto-creates an unprotected one on first use and the secrets have + nowhere to live, so every journey skips. +- **Repository-level would also resolve, and that is the problem.** Environment + secrets are readable only by jobs that enter the environment; repository + secrets are readable by every job, which defeats the isolation. + +**Leave both protection rules off.** Each one breaks a trigger this workflow +actually uses: + +| Rule | Why not | +|------|---------| +| Required reviewers | A `schedule`-triggered job waits for a human instead of running. The cron would never run unattended, and each night queues another pending deployment — which defeats the drift detection the lane exists for. | +| Deployment branches restricted to `main` | On a `pull_request` event `github.ref` is `refs/pull/N/merge`, which does not match `main`, so a `ci:live` run fails with *"Branch is not allowed to deploy to live-llm"*. | + +The protection comes from three properties that hold without either rule: + +1. **Fork pull requests never receive secrets.** GitHub does not pass them for + `pull_request` events from forks, so a fork cannot spend tokens even with the + label applied — the journeys skip instead. +2. **The `ci:live` label requires write or triage permission.** Applying it is the + gate for same-repo pull requests, and anyone able to apply it can already run + `workflow_dispatch`. +3. **Every journey caps its own provider calls and tokens.** Worst-case spend is + bounded by construction, not by trust — see [Cost and Convergence + Guards](#cost-and-convergence-guards). + +### Setting it up in the GitHub UI + +**1. Create the environment** — *Settings → Environments → New environment*, named +`live-llm`. Leave **Required reviewers** unchecked and **Deployment branches** at +*All branches*, for the reasons in the table above. + +**2. Add the three secrets** — on that environment's page, under **Environment +secrets** (not the repository secrets above it), add each row of the table above. +The third one's name is the easy mistake: it is `LEAPFLOW_LLM_CHEAP_MODEL`. + +**3. Verify** — *Actions → Nightly live → Run workflow* (leave `rerecord` false). + +| Result | Meaning | +|--------|---------| +| `3 passed` | Configured correctly | +| `3 skipped` | A secret is missing or misnamed — the log names which: `missing ['LEAPFLOW_LLM_MODEL']` | + +Expanding the *Decide which journeys to run* step shows the selected journeys. + +**Optional — on-demand live runs per pull request.** Create a label named +`ci:live` (*Issues → Labels → New label*). Applying it to a PR runs only the live +journeys that PR's change could affect. Applying a label needs write or triage +permission, which is the gate for same-repo pull requests. + +Measured cost of a full scheduled run against `qwen3.7-plus`: 15–20 provider +calls, ~135k tokens, ~55s. The structural worst case, fixed by the per-journey +ceilings, is 36 calls / ~342k tokens. + +`LEAPFLOW_LLM_CHEAP_MODEL` holds a model name rather than a credential, but the +workflow reads it through `secrets.`, so it has to be stored as a secret to take +effect. Moving it to `vars.` would make it visible and diff-reviewable — and would +remove the name mismatch as a source of confusion — at the cost of editing both +call sites (`nightly-live.yaml` L91 and L137). + +### What deliberately does not need configuring + +`leapd.py` strips **every** inherited `LEAPFLOW_*` from the daemon environment +and injects a fixed set, so extra variables set in CI never reach a journey's +daemon — they only affect the three fallbacks above. + +| Variable | Why it is not a secret | +|----------|------------------------| +| `LEAPFLOW_VLM_*`, `LEAPFLOW_LLM_AUX_*` | Pointed at the same cassette proxy on purpose, so no journey can reach a real provider through a side channel | +| `LEAPFLOW_MOCK_HOST` | Harness sets `1` | +| `LEAPFLOW_DATA_DIR`, `LEAPFLOW_PROFILE`, `LEAPFLOW_LLM_MAX_RETRIES`, `LEAPFLOW_LLM_CONTEXT_LENGTH`, `LEAPFLOW_LOG_LEVEL`, `LEAPFLOW_DAEMON_*` | Harness-controlled per journey | +| `LEAPFLOW_TEST_UPSTREAM_BASE_URL`, `_API_KEY`, `_MODEL` | Optional override, for pointing the recorder at a different endpoint than the one under test | +| `LEAPFLOW_TEST_LLM_MODE` | Set explicitly per job, not a secret | + +### Running the live lane locally + +```bash +export LEAPFLOW_LLM_API_KEY=... +export LEAPFLOW_LLM_BASE_URL=https://.../v1 +export LEAPFLOW_LLM_MODEL=qwen3.7-plus +make test-live # or: make record-traffic +``` + +--- + +## Quick Start + +```bash +# Default test gate (what CI runs) +make test + +# Mock layer only (fast, ~18s) +make test-unit + +# Real journeys only (~7s, offline) +make test-e2e + +# Change-scoped mock layer (local convenience during development) +make test-impact BASE=origin/main + +# Rebuild the offline replay store after changing journey logic +make seed-cassettes +make sync-fixtures +``` diff --git a/tests/_fixtures/cassettes/r1_conversation/cassette-model-76802162bb96917f.cassette.json b/tests/_fixtures/cassettes/r1_conversation/cassette-model-76802162bb96917f.cassette.json new file mode 100644 index 0000000..30c18ee --- /dev/null +++ b/tests/_fixtures/cassettes/r1_conversation/cassette-model-76802162bb96917f.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "76802162bb96917fbd14bb31afcdaa924a31afd51acd4846af9d1bbc2a57dd5e", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Say hello.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Say hello.\nSay hello." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Hello from LeapFlow.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r1_conversation/cassette-model-7e65b23368f3ad75.cassette.json b/tests/_fixtures/cassettes/r1_conversation/cassette-model-7e65b23368f3ad75.cassette.json new file mode 100644 index 0000000..ca8126f --- /dev/null +++ b/tests/_fixtures/cassettes/r1_conversation/cassette-model-7e65b23368f3ad75.cassette.json @@ -0,0 +1,67 @@ +{ + "fingerprint": "7e65b23368f3ad756b40aa2f9d026b9e489fa95b9fff6a810295ee01bfb37373", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-3\n- Original user request: Is that the same invoice?\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Say hello." + }, + { + "role": "assistant", + "content": "Hello from LeapFlow." + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total." + }, + { + "role": "assistant", + "content": "[Called: file_read]\nThe invoice total is 128.50 USD." + }, + { + "role": "user", + "content": "Is that the same invoice?\nIs that the same invoice?" + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "edit_file", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Yes, that is the same invoice.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r1_conversation/cassette-model-99f7b337470a42f2.cassette.json b/tests/_fixtures/cassettes/r1_conversation/cassette-model-99f7b337470a42f2.cassette.json new file mode 100644 index 0000000..44a446f --- /dev/null +++ b/tests/_fixtures/cassettes/r1_conversation/cassette-model-99f7b337470a42f2.cassette.json @@ -0,0 +1,70 @@ +{ + "fingerprint": "99f7b337470a42f2bc8cf89ce1143d60ee93d935c0a224bbe55d0f53d490280e", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Use the file_read tool on invoice.txt and report the total.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello from LeapFlow." + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total.\nUse the file_read tool on invoice.txt and report the total." + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "file_read" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"kind\": \"file_read_evidence\", \"path\": \"\", \"lines\": 2, \"truncated\": false, \"mode\": \"raw\", \"excerpt\": \"Invoice 42\\nTotal: 128.50 USD\", \"start_line\": 1, \"end_line\": 2, \"selected_lines\": 2}", + "tool_result": true + }, + { + "role": "assistant", + "content": "Operation interrupted. Continuing..." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"The invoice total is 128.50 USD.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r1_conversation/cassette-model-bba29ec49d45d0ce.cassette.json b/tests/_fixtures/cassettes/r1_conversation/cassette-model-bba29ec49d45d0ce.cassette.json new file mode 100644 index 0000000..197ab2c --- /dev/null +++ b/tests/_fixtures/cassettes/r1_conversation/cassette-model-bba29ec49d45d0ce.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "bba29ec49d45d0ce6412ba849678f40629c9a362754caa4983615a78b6bd9c0e", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Use the file_read tool on invoice.txt and report the total.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello from LeapFlow." + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total.\nUse the file_read tool on invoice.txt and report the total." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": [{\"id\": \"call_1\", \"type\": \"function\", \"function\": {\"name\": \"file_read\", \"arguments\": \"{\\\"path\\\": \\\"invoice.txt\\\"}\"}}]}, \"finish_reason\": \"tool_calls\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r2_isolation/cassette-model-2d6d85edee8b24c3.cassette.json b/tests/_fixtures/cassettes/r2_isolation/cassette-model-2d6d85edee8b24c3.cassette.json new file mode 100644 index 0000000..03350da --- /dev/null +++ b/tests/_fixtures/cassettes/r2_isolation/cassette-model-2d6d85edee8b24c3.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "2d6d85edee8b24c300666391203a943b194de49b9a64c0e9804cda149a2ed073", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second A turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Workspace A acknowledged." + }, + { + "role": "user", + "content": "Second A turn.\nSecond A turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Still workspace A.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r2_isolation/cassette-model-5df3b81e677c98d6.cassette.json b/tests/_fixtures/cassettes/r2_isolation/cassette-model-5df3b81e677c98d6.cassette.json new file mode 100644 index 0000000..b4c0906 --- /dev/null +++ b/tests/_fixtures/cassettes/r2_isolation/cassette-model-5df3b81e677c98d6.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "5df3b81e677c98d6c3cf2dbad240708a7b39c62dccb628ba7f7e19c8d5cccffd", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second B turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Workspace B acknowledged." + }, + { + "role": "user", + "content": "Second B turn.\nSecond B turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Still workspace B.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json b/tests/_fixtures/cassettes/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json new file mode 100644 index 0000000..dfb485a --- /dev/null +++ b/tests/_fixtures/cassettes/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "6d759c6c949a15c4d57aba6fdcc94eb3f7e9e89e353299eaa4ac1c66ed071a7a", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Hello from A.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Hello from A.\nHello from A." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Workspace A acknowledged.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r2_isolation/cassette-model-d20387e834184f22.cassette.json b/tests/_fixtures/cassettes/r2_isolation/cassette-model-d20387e834184f22.cassette.json new file mode 100644 index 0000000..8cb6dde --- /dev/null +++ b/tests/_fixtures/cassettes/r2_isolation/cassette-model-d20387e834184f22.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "d20387e834184f222f0a3cda065f83f6c0187fbda97c5c3412b01e6645527ccc", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Hello from B.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Hello from B.\nHello from B." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Workspace B acknowledged.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r3_control_plane/cassette-model-466f8fe2c8e497df.cassette.json b/tests/_fixtures/cassettes/r3_control_plane/cassette-model-466f8fe2c8e497df.cassette.json new file mode 100644 index 0000000..41d2db2 --- /dev/null +++ b/tests/_fixtures/cassettes/r3_control_plane/cassette-model-466f8fe2c8e497df.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "466f8fe2c8e497df5654533d8d4781e694e534a97782ba8a60cdf07d6fe1884d", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Anything to report?\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Anything to report?\nAnything to report?" + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Acknowledged.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r4_recovery/cassette-model-0ea1fb58d9fb407a.cassette.json b/tests/_fixtures/cassettes/r4_recovery/cassette-model-0ea1fb58d9fb407a.cassette.json new file mode 100644 index 0000000..57b5714 --- /dev/null +++ b/tests/_fixtures/cassettes/r4_recovery/cassette-model-0ea1fb58d9fb407a.cassette.json @@ -0,0 +1,59 @@ +{ + "fingerprint": "0ea1fb58d9fb407ad9748b096f5a661d7ef8a8ff4c170b4d2a631e7515fbec14", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-3\n- Original user request: Keep going with more context.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Recovered after a server error." + }, + { + "role": "user", + "content": "Keep going with more context.\nKeep going with more context." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 400, + "content_type": "application/json", + "body": "{\"error\": {\"message\": \"This model's maximum context length is 8192 tokens. However, your messages resulted in 9001 tokens.\", \"type\": \"invalid_request_error\", \"code\": \"context_length_exceeded\"}}" + }, + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Recovered after compressing context.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r4_recovery/cassette-model-1010af940122b23a.cassette.json b/tests/_fixtures/cassettes/r4_recovery/cassette-model-1010af940122b23a.cassette.json new file mode 100644 index 0000000..30aad9a --- /dev/null +++ b/tests/_fixtures/cassettes/r4_recovery/cassette-model-1010af940122b23a.cassette.json @@ -0,0 +1,55 @@ +{ + "fingerprint": "1010af940122b23a5bb9238cd9cce1487e0950cf0a9c56196abf10ad3e7f4287", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Summarize the situation.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Summarize the situation.\nSummarize the situation." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 429, + "content_type": "application/json", + "body": "{\"error\": {\"message\": \"Rate limit reached for requests\", \"type\": \"rate_limit_error\", \"code\": \"rate_limit_exceeded\"}}" + }, + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Recovered after a rate limit.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r4_recovery/cassette-model-22cfa61700986a25.cassette.json b/tests/_fixtures/cassettes/r4_recovery/cassette-model-22cfa61700986a25.cassette.json new file mode 100644 index 0000000..378ccef --- /dev/null +++ b/tests/_fixtures/cassettes/r4_recovery/cassette-model-22cfa61700986a25.cassette.json @@ -0,0 +1,59 @@ +{ + "fingerprint": "22cfa61700986a25d1312d3388364f3c1f0f809b7edf67345f10671ce247a238", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: And now?\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Recovered after a rate limit." + }, + { + "role": "user", + "content": "And now?\nAnd now?" + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 500, + "content_type": "application/json", + "body": "{\"error\": {\"message\": \"The server had an error\", \"type\": \"server_error\", \"code\": \"internal_error\"}}" + }, + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Recovered after a server error.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r4_recovery/cassette-model-ffab8956d80a9c5c.cassette.json b/tests/_fixtures/cassettes/r4_recovery/cassette-model-ffab8956d80a9c5c.cassette.json new file mode 100644 index 0000000..ffa77be --- /dev/null +++ b/tests/_fixtures/cassettes/r4_recovery/cassette-model-ffab8956d80a9c5c.cassette.json @@ -0,0 +1,59 @@ +{ + "fingerprint": "ffab8956d80a9c5c25fea82fc9dab04e852d9dc507329393c3156825c915f810", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-4\n- Original user request: Do the impossible thing.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Recovered after compressing context." + }, + { + "role": "user", + "content": "Do the impossible thing.\nDo the impossible thing." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 400, + "content_type": "application/json", + "body": "{\"error\": {\"message\": \"The requested configuration is not supported by this model\", \"type\": \"invalid_request_error\", \"code\": \"unsupported_value\"}}" + }, + { + "status": 400, + "content_type": "application/json", + "body": "{\"error\": {\"message\": \"The requested configuration is not supported by this model\", \"type\": \"invalid_request_error\", \"code\": \"unsupported_value\"}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json b/tests/_fixtures/cassettes/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json new file mode 100644 index 0000000..61a500c --- /dev/null +++ b/tests/_fixtures/cassettes/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "3f3d19a6e4450d6ad6766569d84dd66431abcb255e84a5ffc75ccc0d85f52c6b", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Let me show you something.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Let me show you something.\nLet me show you something." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Noted the first step.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r5_learning/cassette-model-4243a112bcb29cbd.cassette.json b/tests/_fixtures/cassettes/r5_learning/cassette-model-4243a112bcb29cbd.cassette.json new file mode 100644 index 0000000..8ccff8f --- /dev/null +++ b/tests/_fixtures/cassettes/r5_learning/cassette-model-4243a112bcb29cbd.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "4243a112bcb29cbd7863d2c7990d2a401bf64990ae703b6c59c6d04e900b9e95", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-3\n- Original user request: Thanks, that is all.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Noted the second step." + }, + { + "role": "user", + "content": "Thanks, that is all.\nThanks, that is all." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"{\\\"title\\\": \\\"Tidy invoices\\\", \\\"trigger_phrases\\\": [\\\"tidy invoices\\\", \\\"sort invoices\\\"], \\\"steps\\\": [\\\"List the invoice folder\\\", \\\"Classify by month\\\", \\\"Move into folders\\\"], \\\"parameters\\\": [{\\\"name\\\": \\\"path\\\", \\\"description\\\": \\\"invoice folder\\\"}], \\\"pre_conditions\\\": [], \\\"confidence\\\": 0.7}\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r5_learning/cassette-model-c2e43feadd61e3d1.cassette.json b/tests/_fixtures/cassettes/r5_learning/cassette-model-c2e43feadd61e3d1.cassette.json new file mode 100644 index 0000000..5b1dcf2 --- /dev/null +++ b/tests/_fixtures/cassettes/r5_learning/cassette-model-c2e43feadd61e3d1.cassette.json @@ -0,0 +1,62 @@ +{ + "fingerprint": "c2e43feadd61e3d1e70604bdc1ef3f618cea986c6ef76a1b6b785c8883c9c86f", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-3\n- Original user request: Thanks, that is all.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Noted the second step." + }, + { + "role": "user", + "content": "Thanks, that is all.\nThanks, that is all." + }, + { + "role": "assistant", + "content": "{\"title\": \"Tidy invoices\", \"trigger_phrases\": [\"tidy invoices\", \"sort invoices\"], \"steps\": [\"List the invoice folder\", \"Classify by month\", \"Move into folders\"], \"parameters\": [{\"name\": \"path\", \"description\": \"invoice folder\"}], \"pre_conditions\": [], \"confidence\": 0.7}" + }, + { + "role": "user", + "content": "Tool result (path):\n{\"ok\": false, \"error\": \"Unknown tool: path\", \"error_type\": \"unknown_tool\", \"retryable\": true}\nSYSTEM: The previous tool call used an unavailable tool name. Original tool: path. Resolution: unknown (no exact canonical tool name match). Suggested canonical tools: capability_expand, code_intel, code_search, config_get, config_list. Available tools include: capability_expand, code_intel, code_search, config_get, config_list, config_set, delegate_task, edit_file, env_info, file_find, file_list, file_read. Retry once using an exact canonical tool name from the available list and valid arguments. Do not invent tool names, use aliases, or infer a tool from argument shape; answer without a tool if no exact tool fits." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Done \\u2014 nothing further needed.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r5_learning/cassette-model-d10ed33981125492.cassette.json b/tests/_fixtures/cassettes/r5_learning/cassette-model-d10ed33981125492.cassette.json new file mode 100644 index 0000000..142d783 --- /dev/null +++ b/tests/_fixtures/cassettes/r5_learning/cassette-model-d10ed33981125492.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "d10ed339811254925842efddda2fd86f23594b7b3b15c2967507ff2f2e77c9fb", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Now sort them by month.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Noted the first step." + }, + { + "role": "user", + "content": "Now sort them by month.\nNow sort them by month." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Noted the second step.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/cassettes/r6_lifecycle/cassette-model-8b284c8b613c8be3.cassette.json b/tests/_fixtures/cassettes/r6_lifecycle/cassette-model-8b284c8b613c8be3.cassette.json new file mode 100644 index 0000000..8f4bf0f --- /dev/null +++ b/tests/_fixtures/cassettes/r6_lifecycle/cassette-model-8b284c8b613c8be3.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "8b284c8b613c8be3b301712c01c84995acee6ac15d554192185a92ce2a91049f", + "note": "captured in seed mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Are you there?\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Are you there?\nAre you there?" + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"id\": \"chatcmpl-cassette\", \"object\": \"chat.completion\", \"model\": \"cassette-model\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"Still here.\"}, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 64, \"completion_tokens\": 16, \"total_tokens\": 80}}" + } + ] +} diff --git a/tests/_fixtures/llm_responses/response_shapes.json b/tests/_fixtures/llm_responses/response_shapes.json new file mode 100644 index 0000000..595661f --- /dev/null +++ b/tests/_fixtures/llm_responses/response_shapes.json @@ -0,0 +1,154 @@ +{ + "_comment": "Generated by tools/sync_fixtures.py from tests/_fixtures/recordings (real provider traffic) and tests/_fixtures/cassettes (deterministic replay inputs, including injected failures). Do not edit by hand: run `make sync-fixtures` after re-recording.", + "stored_responses_seen": 37, + "completion_shapes": [ + { + "choices": [ + { + "finish_reason": "str", + "index": "int", + "message": { + "content": "str", + "reasoning_content": "str", + "role": "str", + "tool_calls": [ + { + "function": { + "arguments": "str", + "name": "str" + }, + "id": "str", + "index": "int", + "type": "str" + } + ] + } + } + ], + "created": "int", + "id": "str", + "model": "str", + "object": "str", + "usage": { + "completion_tokens": "int", + "completion_tokens_details": { + "reasoning_tokens": "int", + "text_tokens": "int" + }, + "prompt_tokens": "int", + "prompt_tokens_details": { + "cached_tokens": "int", + "text_tokens": "int" + }, + "total_tokens": "int" + } + }, + { + "choices": [ + { + "finish_reason": "str", + "index": "int", + "message": { + "content": "str", + "reasoning_content": "str", + "role": "str" + } + } + ], + "created": "int", + "id": "str", + "model": "str", + "object": "str", + "usage": { + "completion_tokens": "int", + "completion_tokens_details": { + "reasoning_tokens": "int", + "text_tokens": "int" + }, + "prompt_tokens": "int", + "prompt_tokens_details": { + "cached_tokens": "int", + "text_tokens": "int" + }, + "total_tokens": "int" + } + }, + { + "choices": [ + { + "finish_reason": "str", + "index": "int", + "message": { + "content": "str", + "role": "str", + "tool_calls": [ + { + "function": { + "arguments": "str", + "name": "str" + }, + "id": "str", + "type": "str" + } + ] + } + } + ], + "id": "str", + "model": "str", + "object": "str", + "usage": { + "completion_tokens": "int", + "prompt_tokens": "int", + "total_tokens": "int" + } + }, + { + "choices": [ + { + "finish_reason": "str", + "index": "int", + "message": { + "content": "str", + "role": "str" + } + } + ], + "id": "str", + "model": "str", + "object": "str", + "usage": { + "completion_tokens": "int", + "prompt_tokens": "int", + "total_tokens": "int" + } + } + ], + "chunk_shapes": [], + "error_shapes": [ + { + "error": { + "code": "str", + "message": "str", + "type": "str" + } + } + ], + "usage_fields": [ + "completion_tokens", + "completion_tokens_details", + "prompt_tokens", + "prompt_tokens_details", + "total_tokens" + ], + "finish_reasons": [ + "stop", + "tool_calls" + ], + "error_codes": [ + "context_length_exceeded", + "internal_error", + "rate_limit_exceeded", + "unsupported_value" + ] +} diff --git a/tests/_fixtures/recordings/r1_conversation/cassette-model-6aaaff88aaf3bdbd.cassette.json b/tests/_fixtures/recordings/r1_conversation/cassette-model-6aaaff88aaf3bdbd.cassette.json new file mode 100644 index 0000000..01daeb9 --- /dev/null +++ b/tests/_fixtures/recordings/r1_conversation/cassette-model-6aaaff88aaf3bdbd.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "6aaaff88aaf3bdbd6393a14d439a5ebe8032c75907cb3126df72e0e941817bad", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Use the file_read tool on invoice.txt and report the total.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total.\nUse the file_read tool on invoice.txt and report the total." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-1747e2eb-91c2-99cf-929b-598e54714ea7\",\"choices\":[{\"message\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_4a7ddf905f054116a80e4432\",\"type\":\"function\",\"function\":{\"name\":\"file_find\",\"arguments\":\"{\\\"glob\\\": \\\"**/invoice.txt\\\"}\"}}],\"role\":\"assistant\",\"content\":\"\",\"reasoning_content\":\"The user wants me to read invoice.txt and report the total. Let me first find and read the file.\"},\"index\":0,\"finish_reason\":\"tool_calls\"}],\"created\":1785998052,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8792,\"completion_tokens\":55,\"prompt_tokens\":8737,\"completion_tokens_details\":{\"reasoning_tokens\":22,\"text_tokens\":55},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8737}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r1_conversation/cassette-model-76802162bb96917f.cassette.json b/tests/_fixtures/recordings/r1_conversation/cassette-model-76802162bb96917f.cassette.json new file mode 100644 index 0000000..2226edc --- /dev/null +++ b/tests/_fixtures/recordings/r1_conversation/cassette-model-76802162bb96917f.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "76802162bb96917fbd14bb31afcdaa924a31afd51acd4846af9d1bbc2a57dd5e", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Say hello.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Say hello.\nSay hello." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-8780c4bb-be8c-9a81-b39b-284219a459e9\",\"choices\":[{\"message\":{\"content\":\"Hello! 👋 How can I help you today?\",\"reasoning_content\":\"The user is simply saying hello. This is a straightforward greeting that doesn't require any tool usage. I should respond warmly and naturally.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1785998050,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8734,\"completion_tokens\":43,\"prompt_tokens\":8691,\"completion_tokens_details\":{\"reasoning_tokens\":27,\"text_tokens\":43},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8691}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r1_conversation/cassette-model-a7f9c8dac3af2ca7.cassette.json b/tests/_fixtures/recordings/r1_conversation/cassette-model-a7f9c8dac3af2ca7.cassette.json new file mode 100644 index 0000000..83d8c20 --- /dev/null +++ b/tests/_fixtures/recordings/r1_conversation/cassette-model-a7f9c8dac3af2ca7.cassette.json @@ -0,0 +1,82 @@ +{ + "fingerprint": "a7f9c8dac3af2ca7bb930c3a00700742b0ed1910b6104aa8dfd6acd8b4a5af16", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Use the file_read tool on invoice.txt and report the total.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total.\nUse the file_read tool on invoice.txt and report the total." + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "file_find" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"glob\": \"**/invoice.txt\", \"path\": \"\", \"files\": [\"\"], \"file_count\": 1, \"truncated\": false, \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "file_read" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"kind\": \"file_read_evidence\", \"path\": \"\", \"lines\": 2, \"truncated\": false, \"mode\": \"raw\", \"excerpt\": \"Invoice 42\\nTotal: 128.50 USD\", \"start_line\": 1, \"end_line\": 2, \"selected_lines\": 2}", + "tool_result": true + }, + { + "role": "assistant", + "content": "Operation interrupted. Continuing..." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-f8efdc88-d99d-92b8-8cb9-043d643c50f3\",\"choices\":[{\"message\":{\"content\":\"The invoice shows a total of **128.50 USD**.\",\"reasoning_content\":\"The user wants me to read invoice.txt and report the total. I already found and read the file in previous turns. The file contains:\\n\\n```\\nInvoice 42\\nTotal: 128.50 USD\\n```\\n\\nSo the total is 128.50 USD. I should report this directly without making additional tool calls since I already have the information.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1785998056,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":9259,\"completion_tokens\":99,\"prompt_tokens\":9160,\"completion_tokens_details\":{\"reasoning_tokens\":79,\"text_tokens\":99},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":9160}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r1_conversation/cassette-model-a8010d7e372fea10.cassette.json b/tests/_fixtures/recordings/r1_conversation/cassette-model-a8010d7e372fea10.cassette.json new file mode 100644 index 0000000..60da316 --- /dev/null +++ b/tests/_fixtures/recordings/r1_conversation/cassette-model-a8010d7e372fea10.cassette.json @@ -0,0 +1,70 @@ +{ + "fingerprint": "a8010d7e372fea105f4bffeb50edd654f791fd6e4a4773c3fa46c7d0634955d6", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Use the file_read tool on invoice.txt and report the total.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Use the file_read tool on invoice.txt and report the total.\nUse the file_read tool on invoice.txt and report the total." + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "file_find" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"glob\": \"**/invoice.txt\", \"path\": \"\", \"files\": [\"\"], \"file_count\": 1, \"truncated\": false, \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "assistant", + "content": "Operation interrupted. Continuing..." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-a21e1fb2-6205-9232-9b79-130ced9620db\",\"choices\":[{\"message\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_52d85fa234044982a9aaefc7\",\"type\":\"function\",\"function\":{\"name\":\"file_read\",\"arguments\":\"{\\\"path\\\": \\\"/private/tmp/lfj-r1_conversation/workspaces/main/invoice.txt\\\"}\"}}],\"role\":\"assistant\",\"content\":\"\",\"reasoning_content\":\"The user wants me to read invoice.txt and report the total. I found the file at /private/tmp/lfj-r1_conversation/workspaces/main/invoice.txt. Now I need to read it.\"},\"index\":0,\"finish_reason\":\"tool_calls\"}],\"created\":1785998054,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":9096,\"completion_tokens\":85,\"prompt_tokens\":9011,\"completion_tokens_details\":{\"reasoning_tokens\":42,\"text_tokens\":85},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":9011}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-0eee36f0747e221b.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-0eee36f0747e221b.cassette.json new file mode 100644 index 0000000..0527949 --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-0eee36f0747e221b.cassette.json @@ -0,0 +1,76 @@ +{ + "fingerprint": "0eee36f0747e221b0f1db1682d1099344efee0707b2ccb4a23b2c3ab5e30842d", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second A turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Second A turn.\nSecond A turn." + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "repo_map", + "memory_search" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"tool\": \"repo_map\", \"root\": \"\", \"languages\": [], \"test_command\": \"\", \"lint_command\": \"\", \"structure\": {\"dirs\": [], \"files\": []}, \"entry_points\": [], \"manifest\": {}, \"readme\": \"\", \"vcs\": {\"git\": false}, \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "tool", + "content": "{\"ok\": true, \"result\": \"{\\\"results\\\": []}\", \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "assistant", + "content": "Operation interrupted. Continuing..." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-d5aef213-cbca-9ca7-bfe5-de937bc0c07e\",\"choices\":[{\"message\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_000ebc8b56f6431583c3d199\",\"type\":\"function\",\"function\":{\"name\":\"file_list\",\"arguments\":\"{\\\"path\\\": \\\"/private/tmp/lfj-r2_isolation/workspaces/alpha\\\", \\\"depth\\\": 2}\"}},{\"index\":1,\"id\":\"call_4d93feb3a6ce4f298786bc94\",\"type\":\"function\",\"function\":{\"name\":\"memory_search\",\"arguments\":\"{\\\"query\\\": \\\"alpha workspace task\\\"}\"}}],\"role\":\"assistant\",\"content\":\"\",\"reasoning_content\":\"The user says \\\"Second A turn.\\\" The workspace is empty (no files, no git, no languages detected). There's no prior memory about this. The message is ambiguous—it could be a test prompt, a placeholder, or something with context I'm missing.\\n\\nLet me check if there's anything in the workspace at all, and also check for any session history that might clarify what \\\"Second A turn\\\" means.\"},\"index\":0,\"finish_reason\":\"tool_calls\"}],\"created\":1786001263,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":9354,\"completion_tokens\":166,\"prompt_tokens\":9188,\"completion_tokens_details\":{\"reasoning_tokens\":85,\"text_tokens\":166},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":9188}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-22422bffdc9a2b14.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-22422bffdc9a2b14.cassette.json new file mode 100644 index 0000000..abb176d --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-22422bffdc9a2b14.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "22422bffdc9a2b147773fafb1cb6d4604129ca206d45ef166766c72cfd66396c", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second B turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello, B! 👋 Welcome — how can I help you today?" + }, + { + "role": "user", + "content": "Second B turn.\nSecond B turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-085f2e88-9237-9762-9339-2cc2f36b6e8c\",\"choices\":[{\"message\":{\"content\":\"It looks like you're testing things out! I'm here and ready whenever you have a question or task. What can I help you with?\",\"reasoning_content\":\"The user is saying \\\"Second B turn.\\\" twice. This seems like a test or placeholder message. I should respond naturally and ask how I can help.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001260,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8781,\"completion_tokens\":65,\"prompt_tokens\":8716,\"completion_tokens_details\":{\"reasoning_tokens\":31,\"text_tokens\":65},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8716}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-332c6e9b627ea516.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-332c6e9b627ea516.cassette.json new file mode 100644 index 0000000..28f10f9 --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-332c6e9b627ea516.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "332c6e9b627ea5162f541caaed2d74970c812af5269f85537a8d68e7bc1370f1", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second A turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Second A turn.\nSecond A turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-b1fd9bf9-3176-9e7f-9748-3ba8494a11ea\",\"choices\":[{\"message\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_9cef3d70ddf7456c890f0eaf\",\"type\":\"function\",\"function\":{\"name\":\"repo_map\",\"arguments\":\"{\\\"path\\\": \\\"/private/tmp/lfj-r2_isolation/workspaces/alpha\\\"}\"}},{\"index\":1,\"id\":\"call_40599ab79a3541b0a4da784a\",\"type\":\"function\",\"function\":{\"name\":\"memory_search\",\"arguments\":\"{\\\"query\\\": \\\"Second A turn\\\"}\"}}],\"role\":\"assistant\",\"content\":\"\",\"reasoning_content\":\"The user's message is \\\"Second A turn.\\\" repeated twice. This seems like a test message or possibly a continuation of some experiment. Let me check if there's any context in memory or the workspace that might explain what this means.\\n\\nLet me look at the workspace to understand the context.\"},\"index\":0,\"finish_reason\":\"tool_calls\"}],\"created\":1786001260,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8841,\"completion_tokens\":129,\"prompt_tokens\":8712,\"completion_tokens_details\":{\"reasoning_tokens\":59,\"text_tokens\":129},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8712}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-39bd3e1d47a371ad.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-39bd3e1d47a371ad.cassette.json new file mode 100644 index 0000000..d435a1c --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-39bd3e1d47a371ad.cassette.json @@ -0,0 +1,94 @@ +{ + "fingerprint": "39bd3e1d47a371ad569f58d47dd59c66922f6ab5ee4a877de229003f332bfae2", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second A turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Second A turn.\nSecond A turn." + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "repo_map", + "memory_search" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"tool\": \"repo_map\", \"root\": \"\", \"languages\": [], \"test_command\": \"\", \"lint_command\": \"\", \"structure\": {\"dirs\": [], \"files\": []}, \"entry_points\": [], \"manifest\": {}, \"readme\": \"\", \"vcs\": {\"git\": false}, \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "tool", + "content": "{\"ok\": true, \"result\": \"{\\\"results\\\": []}\", \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + "file_list", + "memory_search" + ] + }, + { + "role": "tool", + "content": "{\"ok\": true, \"kind\": \"file_list_evidence\", \"path\": \"\", \"depth\": 2, \"tree\": [], \"total_entries\": 0, \"truncated\": false}", + "tool_result": true + }, + { + "role": "tool", + "content": "{\"ok\": true, \"result\": \"{\\\"results\\\": []}\", \"execution_id\": \"\", \"idempotency_key\": \"\", \"execution_policy\": \"read_only\", \"tool_call_id\": \"\", \"execution_status\": \"completed\"}", + "tool_result": true + }, + { + "role": "assistant", + "content": "Operation interrupted. Continuing..." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-98977f50-67f4-9b0c-8ac3-2500619e5315\",\"choices\":[{\"message\":{\"content\":\"I've checked the workspace and it's currently empty — no files or projects are present in `/private/tmp/lfj-r2_isolation/workspaces/alpha`.\\n\\nWhat would you like me to work on? I can help with:\\n\\n- **Creating files or projects** from scratch\\n- **Analyzing code** if you add files to the workspace\\n- **Running commands** or scripts\\n- **Research tasks** or information gathering\\n- **Any other task** you have in mind\\n\\nJust let me know what you'd like to accomplish!\",\"reasoning_content\":\"The user said \\\"Second A turn.\\\" This appears to be a test message or a placeholder message. The workspace is empty (no files, no git repository). There's no prior context in memory about what \\\"Second A turn\\\" means.\\n\\nLet me check if there are any other workspaces or if this is part of a multi-workspace test scenario. The task contract mentions:\\n- Task ID: turn-2\\n- Original user request: Second A turn.\\n- Workspace root: /private/tmp/lfj-r2_isolation/workspaces/alpha\\n\\nThis suggests this is \\\"turn-2\\\" of some kind of test or evaluation. The workspace is empty, so there's no code to work with. The message \\\"Second A turn\\\" seems like it might be a test prompt to see how I respond when given minimal context.\\n\\nSince the workspace is empty and there's no clear task, I should respond acknowledging that I'm ready to help, but note that the workspace appears to be empty. I should ask what the user would like me to do.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001266,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":9824,\"completion_tokens\":334,\"prompt_tokens\":9490,\"completion_tokens_details\":{\"reasoning_tokens\":217,\"text_tokens\":334},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":9490}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-651bca850f00e95e.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-651bca850f00e95e.cassette.json new file mode 100644 index 0000000..f0d83b7 --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-651bca850f00e95e.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "651bca850f00e95e31aac386be84817b94896ac921c62a123184e49c32bd2dc9", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second B turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello B! 👋 Welcome — how can I help you today?" + }, + { + "role": "user", + "content": "Second B turn.\nSecond B turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-be63f842-f3f4-918c-aae3-76df07c4d4bf\",\"choices\":[{\"message\":{\"content\":\"Hello again! It looks like your message might be a test or placeholder — there's no specific task or question included.\\n\\nHow can I help you today? Whether it's exploring the workspace, writing code, answering questions, or anything else, just let me know!\",\"reasoning_content\":\"The user is saying \\\"Second B turn. Second B turn.\\\" This appears to be a test or placeholder message, similar to the task context which mentions \\\"Second B turn.\\\" as the original user request. There's no actual task or question here - it seems like a system test or placeholder.\\n\\nI should respond naturally and ask how I can help, since there's no concrete request to act on.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001249,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8855,\"completion_tokens\":140,\"prompt_tokens\":8715,\"completion_tokens_details\":{\"reasoning_tokens\":81,\"text_tokens\":140},\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":8715}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json new file mode 100644 index 0000000..313657b --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-6d759c6c949a15c4.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "6d759c6c949a15c4d57aba6fdcc94eb3f7e9e89e353299eaa4ac1c66ed071a7a", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Hello from A.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Hello from A.\nHello from A." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-4dc7aae8-7423-9d23-994e-f27b9b8e3b3d\",\"choices\":[{\"message\":{\"content\":\"Hello! 👋 How can I help you today?\",\"reasoning_content\":\"The user is saying hello. This is a simple greeting, so I should respond naturally without using any tools. The task contract mentions \\\"Hello from A\\\" as the original user request, and the workspace is set up at /private/tmp/lfj-r2_isolation/workspaces/alpha.\\n\\nSince this is just a greeting, I should respond warmly and ask how I can help them. No tools are needed here.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001255,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8797,\"completion_tokens\":101,\"prompt_tokens\":8696,\"completion_tokens_details\":{\"reasoning_tokens\":85,\"text_tokens\":101},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8696}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-d20387e834184f22.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-d20387e834184f22.cassette.json new file mode 100644 index 0000000..608eab1 --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-d20387e834184f22.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "d20387e834184f222f0a3cda065f83f6c0187fbda97c5c3412b01e6645527ccc", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Hello from B.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Hello from B.\nHello from B." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-c556c42d-3f81-92d1-9b6f-e428836e90e5\",\"choices\":[{\"message\":{\"content\":\"Hello, B! 👋 Welcome — how can I help you today?\",\"reasoning_content\":\"The user is saying \\\"Hello from B.\\\" twice. This is a simple greeting, so I should respond warmly and directly without needing any tools.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001258,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8745,\"completion_tokens\":49,\"prompt_tokens\":8696,\"completion_tokens_details\":{\"reasoning_tokens\":29,\"text_tokens\":49},\"prompt_tokens_details\":{\"cached_tokens\":8448,\"text_tokens\":8696}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r2_isolation/cassette-model-ee1be6463c408464.cassette.json b/tests/_fixtures/recordings/r2_isolation/cassette-model-ee1be6463c408464.cassette.json new file mode 100644 index 0000000..79b0c3f --- /dev/null +++ b/tests/_fixtures/recordings/r2_isolation/cassette-model-ee1be6463c408464.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "ee1be6463c408464fdc9ed1895e530b686e6f8512168fcd5a0e24b34aae1df2a", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Second A turn.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Hello from A! 👋 How can I help you today?" + }, + { + "role": "user", + "content": "Second A turn.\nSecond A turn." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-36e11737-0bea-9806-9640-200a4074389a\",\"choices\":[{\"message\":{\"content\":\"Still here! Is there something specific you'd like me to help with? I can work with files, search code, run commands, check git history, or anything else in this workspace.\",\"reasoning_content\":\"The user is saying \\\"Second A turn.\\\" twice. This seems like a test or continuation of the previous exchange where I said \\\"Hello from A!\\\" They might be testing if I'm still responsive or if there's something specific they want me to do. Let me just respond naturally and ask what they need.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001249,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8819,\"completion_tokens\":105,\"prompt_tokens\":8714,\"completion_tokens_details\":{\"reasoning_tokens\":62,\"text_tokens\":105},\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":8714}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r5_learning/cassette-model-1261ca61356ab7dd.cassette.json b/tests/_fixtures/recordings/r5_learning/cassette-model-1261ca61356ab7dd.cassette.json new file mode 100644 index 0000000..c06d7cd --- /dev/null +++ b/tests/_fixtures/recordings/r5_learning/cassette-model-1261ca61356ab7dd.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "1261ca61356ab7ddf3fee004383761fd1482474ba80efd4cc7fd00a49fe0f502", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-3\n- Original user request: Thanks, that is all.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "I'd be happy to help sort things by month! However, I don't see any items or data in our conversation yet. Could you share what you'd like me to sort? For example:\n\n- A list of dates or events\n- Files in a directory\n- Data from a file or spreadsheet\n- Something else?\n\nOnce you provide the items, I can sort them by month for you." + }, + { + "role": "user", + "content": "Thanks, that is all.\nThanks, that is all." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-6c7d8a3e-a6ef-9e24-ab3f-2324083ad039\",\"choices\":[{\"message\":{\"content\":\"You're welcome! Feel free to reach out anytime if you need help with sorting, organizing, or anything else. Have a great day!\",\"reasoning_content\":\"The user is saying \\\"Thanks, that is all.\\\" This appears to be a closing message indicating they don't need any further assistance. They're essentially ending the conversation.\\n\\nI should respond politely and briefly, acknowledging their thanks and offering future help if needed. No tools are needed here - this is just a conversational closing.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001283,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8887,\"completion_tokens\":99,\"prompt_tokens\":8788,\"completion_tokens_details\":{\"reasoning_tokens\":66,\"text_tokens\":99},\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":8788}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r5_learning/cassette-model-349252e71fb3aa07.cassette.json b/tests/_fixtures/recordings/r5_learning/cassette-model-349252e71fb3aa07.cassette.json new file mode 100644 index 0000000..ea962ae --- /dev/null +++ b/tests/_fixtures/recordings/r5_learning/cassette-model-349252e71fb3aa07.cassette.json @@ -0,0 +1,54 @@ +{ + "fingerprint": "349252e71fb3aa0702578acfdc9c6b65e29a2fc1f193fa482f297c9c6904317b", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-2\n- Original user request: Now sort them by month.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "assistant", + "content": "Of course! I'm all ears — go ahead and share what you'd like to show me." + }, + { + "role": "user", + "content": "Now sort them by month.\nNow sort them by month." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-9547fd5a-54df-9307-ba8f-b70e1af92be3\",\"choices\":[{\"message\":{\"content\":\"I'd be happy to help sort things by month! However, I don't see any items or data in our conversation yet. Could you share what you'd like me to sort? For example:\\n\\n- A list of dates or events\\n- Files in a directory\\n- Data from a file or spreadsheet\\n- Something else?\\n\\nOnce you provide the items, I can sort them by month for you.\",\"reasoning_content\":\"The user is asking me to \\\"sort them by month.\\\" This implies there's some existing context about items that need to be sorted. However, looking at the conversation history, I don't see any previous messages where items were listed or discussed. The user says \\\"Now sort them by month\\\" but there's no prior context about what \\\"them\\\" refers to.\\n\\nI should ask the user to clarify what they want me to sort, since I don't have context about what items they're referring to.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001278,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8915,\"completion_tokens\":190,\"prompt_tokens\":8725,\"completion_tokens_details\":{\"reasoning_tokens\":102,\"text_tokens\":190},\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":8725}}}" + } + ] +} diff --git a/tests/_fixtures/recordings/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json b/tests/_fixtures/recordings/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json new file mode 100644 index 0000000..d24de0d --- /dev/null +++ b/tests/_fixtures/recordings/r5_learning/cassette-model-3f3d19a6e4450d6a.cassette.json @@ -0,0 +1,50 @@ +{ + "fingerprint": "3f3d19a6e4450d6ad6766569d84dd66431abcb255e84a5ffc75ccc0d85f52c6b", + "note": "captured in record mode", + "request": { + "model": "cassette-model", + "stream": false, + "messages": [ + { + "role": "system", + "content": "You are LeapFlow, an intelligent assistant that can both converse naturally and take real actions on the user's computer.\n\n## Capabilities\nThe tool index below lists **every** registered tool by name and a one-line summary — this is the complete\ncapability contract; nothing else exists. Only a subset is directly callable this turn (via native tool calling,\nnot a JSON block in your reply). If you need a tool from the index that is not yet callable, call\n`capability_expand` with its category name first — the matching tools become callable immediately after.\n- **file_list**(path, pattern, depth): List files and directories at a given path. Use depth=1 or depth=2 to get a recursive tree in one call instead of listing each sub-directory separately.\n- **file_read**(path, max_lines, start_line, max_chars, mode): Read text file content with adaptive context governance. For large or unfamiliar files, prefer mode='outline' or mode='symbols' first, then use mode='raw' with start_line/max_lines for the specific range you actually need. For LeapFlow's own settings, use config_list / config_get / config_set — its config files are outside the workspace and not readable here.\n- **file_write**(path, content, mode) [capability_expand category: write]: Write content to a file (overwrite or append).\n- **code_search**(pattern, patterns, path, glob, ignore_case, multiline, max_results, context_lines): Search file CONTENTS by regex across a directory tree (ripgrep-backed). Prefer this over shell_run grep: faster, skips VCS/dependency/build dirs, and returns structured path:line:column matches. Batch related lookups into ONE call via `patterns` (OR-combined, single pass) instead of issuing several separate searches. Use file_read for the surrounding context of a hit.\n- **file_find**(glob, path, max_results): Find files by a recursive glob pattern under a base path (e.g. '**/test_*.py' or '*.md'). Prefer this over shell_run find; skips VCS/dependency/build dirs.\n- **edit_file**(path, edits, dry_run, diff) [capability_expand category: file]: Apply targeted, anchored search-replace edits to an EXISTING text file (use file_write to create/overwrite). Each edit is {original_text, new_text, replace_all?}; original_text must match exactly and uniquely (or set replace_all) — a non-unique or missing anchor is rejected so files are never corrupted. Set dry_run to preview. Alternatively pass a unified 'diff' to apply its hunks as anchored edits. Far cheaper and safer than rewriting a whole file.\n- **code_intel**(path, operation): Precise document symbols (outline) for a source file: classes, functions, and methods with line ranges. Python uses an exact AST parse; other languages use a keyword-prefix scan. Prefer over file_read mode=symbols for accurate navigation before editing. Read-only.\n- **repo_map**(path): Compact project orientation for a repository root: languages, detected test/lint commands, top-level structure, entry points, manifest, and VCS branch. Call this first when entering an unfamiliar codebase. Read-only.\n- **shell_run**(command, cwd, timeout) [capability_expand category: shell]: Execute a one-shot shell command with timeout protection. Runs in the active workspace; paths resolving outside it are refused. Reach for a structured tool first when one fits — web_fetch for anything over HTTP(S), git_query/scm_sync for git, code_search/file_find/file_read for the repo, config_get/config_set for LeapFlow's own settings — because those report typed results, while a failed shell command can only be diagnosed from its exit code and stderr. Every shell run counts as an external side effect, so a failure stops the rest of the batch and is not retried automatically.\n- **scm_sync**(action, cwd, remote, pull_ref, push_ref, timeout) [capability_expand category: scm]: Run a typed git SCM action. Use this instead of shell_run for git pull/push/status. For 'pull origin main then push', set action='pull_then_push', remote='origin', pull_ref='main', and omit push_ref so LeapFlow pushes the current local branch.\n- **git_query**(action, cwd, ref, path, staged, max_count, stat): Read-only structured git inspection: action=diff|log|status|branch|show. Prefer over shell_run for reading repo state — output is clipped, redacted, and log/branch are parsed into structured fields. Use scm_sync for pull/push.\n- **git_write**(action, cwd, message, stage_all, name, ref, create) [capability_expand category: scm]: Mutating git actions: action=commit (message, stage_all), branch (create+switch), checkout (switch; create=true for -b). Approval-gated. Use scm_sync for pull/push and git_query for reads.\n- **time_get**(): Get current date and time.\n- **env_info**(): Get system environment information (OS, Python version, cwd).\n- **text_search**(text, pattern): Search for a regex pattern in text.\n- **text_replace**(text, old, new, count) [capability_expand category: write]: Replace occurrences of a substring in text.\n- **skills_list**(query, category, source): List available learned skills. Use when user asks about capabilities or you need a specific skill.\n- **skill_view**(name): View the full content of a specific skill document.\n- **memory_search**(query, limit): Search agent memory for relevant past experiences, observations, and facts.\n- **memory_add**(content, kind) [capability_expand category: write]: Store a new observation or insight in memory for future reference.\n- **research_note**(kind, text): Record a compact, structured note about the current task's state so it survives context compression on long / multi-step tasks. Use for durable findings, open questions still to resolve, decisions / excluded paths, and the immediate next step. One concise sentence per note.\n- **schedule_reentry**(kind, reason, delay_seconds, event_match, max_reentries, deadline_seconds): Register a re-entry so this task can resume later from its current orientation (findings / open questions / next step). Use when work must pause and continue after a delay (kind=time) or when a matching platform event arrives (kind=event), instead of finishing now. The research-ledger state is carried over automatically.\n- **capability_expand**(category): Fetch the full callable schema for every tool in a capability category. Current non-core categories that require expansion: config, delegate, file, gateway, hub, scm, shell, terminal, write. The compact tool index always lists every registered tool by name and a one-line summary tagged with its exact capability_expand category, but only a static low-risk subset is directly callable each turn. If you need a tool from the index that is not yet callable, call capability_expand with the exact category shown next to it; the matching tools become callable in this turn. Never invent a tool name — expand the category instead.\n- **delegate_task**(goal, context) [capability_expand category: delegate]: Delegate a complex sub-task to an isolated subagent. The subagent gets a fresh context and restricted tool access. Use when a task is self-contained and can be solved independently.\n- **test_run**(command, cwd, timeout): Run the project's test suite and return structured results (framework, passed/failed counts, failing tests). Auto-detects the runner (pytest/npm/go/cargo) or uses a configured/explicit command; executes via the governed shell. ok=true means the runner executed — see 'success' for pass/fail.\n- **lint_check**(command, cwd, timeout): Run the project's linter and return a structured clean/issue result. Auto-detects the linter (ruff/eslint/go vet/clippy) or uses a configured/explicit command; executes via the governed shell. ok=true means the linter ran — see 'clean'.\n- **terminal_open**(command, cwd, shell) [capability_expand category: terminal]: Open a PERSISTENT shell session (REPL/dev server/watch), returning a session_id for terminal_send/read/close. Disabled unless tools.terminal_session_enabled is set. For one-shot commands use shell_run instead.\n- **terminal_send**(session_id, input, wait) [capability_expand category: terminal]: Send a line of input to a persistent terminal session and return output captured shortly after.\n- **terminal_read**(session_id, wait): Drain buffered output from a persistent terminal session (optionally waiting briefly first).\n- **terminal_close**(session_id) [capability_expand category: terminal]: Terminate a persistent terminal session and release its process group.\n- **terminal_list**(): List active persistent terminal sessions.\n- **config_list**(category, limit): List LeapFlow's own writable settings (model, provider, daemon, memory, perception, gateway, …) with current values. Use this to discover the exact key before changing anything. Optionally narrow by `category`. This is the only correct way to inspect LeapFlow configuration — never read config files from disk.\n- **config_get**(key): Read one LeapFlow setting by key (e.g. 'llm.model', 'daemon.log_level'), returning its current value, type, scopes, and whether a change needs a daemon restart. Never read LeapFlow config files from disk — use this.\n- **config_set**(key, value, scope) [capability_expand category: config]: Change one LeapFlow setting by key, e.g. switch the model with key='llm.model'. Values are validated and coerced; credentials are stored in the vault automatically. Call config_list or config_get first if unsure of the key. The result states whether a `leap daemon restart` is required. Never edit LeapFlow config files directly.\n- **web_fetch**(url, select, timeout, max_bytes): Read a URL over HTTP(S) and get back extracted, context-sized content: parsed JSON for API endpoints, readable text plus links for web pages. Use this for anything on the internet — prices, docs, releases, articles — instead of running curl through shell_run: it reports real HTTP status codes, retries rate limits on its own, and is a plain read so a retry is always safe. For JSON APIs pass `select` with a dotted path (e.g. 'chart.result.0.meta') to return just that part instead of the whole payload.\n- **hub_push**(skill_name, visibility, version) [capability_expand category: hub]: Push a local skill to the ModelScope Hub for sharing or backup.\n- **hub_pull**(repo_id, version) [capability_expand category: hub]: Pull a skill from the ModelScope Hub to install locally.\n- **hub_search**(query) [capability_expand category: hub]: Search for skills on the Hub by keyword or description.\n- **hub_sync**(mode, dry_run) [capability_expand category: hub]: Preview or execute sync between local skills and Hub.\n- **platform_action**(platform, action, payload, backend_kind) [capability_expand category: gateway]: Execute an exact registered business action on an external platform through LeapFlow's App Connector layer. Actions must be copied from the App Connector Capability Index and are addressed as domain.operation, e.g. im.send_message or docs.create_markdown. All business fields (chat_id, text, query, etc.) MUST be placed inside `payload`, never at the top level. Example: {\"platform\":\"feishu\",\"action\":\"im.send_message\",\"payload\":{\"chat_id\":\"oc_xxx\",\"text\":\"hello\"}}. Do not invent action names, do not use management actions such as list/guide/connect/status here.\n- **platform_connect**(action, platform, credentials, options, checkpoint) [capability_expand category: gateway]: List, guide, connect, disconnect, remove, or check status for external platforms using the App Connector management namespace. Supports REST and CLI backends. Use this for management actions such as list/guide/preflight/connect/status; use platform_action only for exact registered business actions.\n- **gateway_send**(platform, chat_id, text, thread_id) [capability_expand category: gateway]: Send a message to a connected external platform (Feishu group, Telegram chat, DingTalk conversation, etc.). Requires the platform to be connected via gateway_connect first. Use gateway_connect with action='list' to see connected platforms and available chat IDs.\n- **gateway_connect**(action, platform, credentials, options) [capability_expand category: gateway]: Connect, configure, or manage external platform integrations (Feishu, DingTalk, Telegram, Slack, Discord, etc.). Conversational flow: 1) call 'guide' to get setup steps + required fields, 2) present the steps to the user and ask for ALL required credentials in a single message, 3) call 'connect' with the credentials. Goal: complete in 1–2 user turns. NEVER include credential values in your text response.\n- **session_search**(query, limit) [capability_expand category: unclassified]: Search past conversation sessions for relevant context.\n\n## App Connector Capability Index\nLeapFlow can onboard and manage external apps through `platform_connect` and execute exact registered business actions through `platform_action`.\nFor requests about connecting, setting up, configuring, enabling, or managing a supported app, use `platform_connect` first instead of generating SDK/Webhook sample code.\n`platform_connect.action` is the management namespace: list, guide, preflight, connect, disconnect, remove, status, events_start, events_stop, events_status.\n`platform_action.action` is only for exact registered platform business actions listed below, such as `im.send_message`; never use management actions like `list` or `guide` there.\nAll business fields MUST go inside `payload`; top-level keys are only `platform`, `action`, and `payload`.\nDo not invent platform IDs or platform action names. If the needed action is not listed, ask for discovery/clarification instead of guessing.\nUse `platform_connect` with `action='guide'` and the matching `platform` to start onboarding; use `action='list'` when the app is unclear.\nWhen a pending onboarding state is present, continue from that state with `platform_connect` instead of asking the user to restate the app.\nSupported apps:\n- `api_server`: API Server (OpenAI Compatible) (category=api; backend=adapter; platform_action actions=none registered)\n- `dingtalk`: 钉钉 (DingTalk) (category=im; backend=adapter; platform_action actions=none registered)\n- `feishu`: 飞书 (Feishu/Lark) (category=im; backend=cli)\n - `calendar.create_event` payload={summary*, start_time*, end_time*, attendees} [write/high]\n - `docs.create_markdown` payload={title*, markdown*, folder_token} [write/medium]\n - `drive.search` payload={query*, limit} [read/medium]\n - `im.add_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.download_resource` payload={message_id*, file_key*, type*} [read/low]\n - `im.get_messages` payload={message_ids*} [read/low]\n - `im.list_chats` payload={page_size} [read/low]\n - `im.list_messages` payload={chat_id*} [read/low]\n - `im.list_thread_messages` payload={thread*} [read/low]\n - `im.remove_reaction` payload={message_id*, emoji_type*} [write/low]\n - `im.reply_message` payload={message_id*, text*} [send/high]\n - `im.search_chats` payload={query*, page_size} [read/low]\n - `im.search_messages` payload={query*} [read/medium]\n - `im.send_message` payload={chat_id*, text*, thread_id} [send/high]\n - `im.update_card` payload={token*, card*} [write/medium]\n - `im.update_message` payload={message_id*, text*} [write/medium]\n - `mail.search_unread` payload={query, limit} [read/high]\n - `sheets.append_row` payload={spreadsheet_token*, sheet_id*, values*} [write/medium]\n - `task.create` payload={title*, description, due_time} [write/medium]\n- `telegram`: Telegram (category=social; backend=adapter; platform_action actions=none registered)\n- `webhook`: Webhook (Generic) (category=webhook; backend=adapter; platform_action actions=none registered)\n\n## Tool Usage\nTools are normally invoked through the native function-calling mechanism, not by writing JSON in your reply\ntext. Only if the provider signals that native function calling is unavailable for this turn, fall back to a\nsingle JSON code block: `{\"name\": \"tool_name\", \"arguments\": {\"key\": \"value\"}}` — use this fallback format\nonly, never both. Only call a tool whose exact name appears in the tool index above and is currently callable\n(or reachable via `capability_expand`). Never invent, rename, alias, or guess a tool name, platform ID, or\nplatform action from argument shape or wording — if the index does not list it, it does not exist.\n`platform_connect.action` (list/guide/preflight/connect/disconnect/remove/status/events_start/events_stop/\nevents_status) is the App Connector management namespace; `platform_action.action` only accepts exact\nregistered business actions such as `im.send_message` shown in the App Connector Capability Index — never mix\nthe two namespaces. If a tool call returns an unknown/unavailable result, use the returned suggestions or\navailable names for a single retry instead of trying further variations of the same guess.\n\n**Side-effect action rule** (`platform_action` with effect=send/write/execute):\n- Call each unique action+payload **exactly once**. Never include duplicates in the same turn.\n- Once the result returns `\"completed\": true`, that action is DONE for this task. Do NOT call it again\n in any subsequent turn — immediately summarize the result for the user instead.\n- The system enforces idempotency: duplicate calls are blocked and will not execute.\n- If the user explicitly requests sending/writing multiple times, use distinct payloads per call.\n\n**Resource identifier provenance rule**:\n- NEVER fabricate, guess, or infer resource identifiers (chat_id, message_id, file_key, user_id, etc.).\n Every resource ID used in a side-effect action MUST come from a successful API response in this session.\n- If a read/list action fails (e.g. authorization error), you do NOT have valid resource IDs.\n Report the failure to the user — do NOT attempt the dependent write/send action with a guessed ID.\n- When a tool result contains `\"llm_instruction\"`, follow it exactly.\n\n## Guidelines\n1. **Direct answers first**: If you already know the answer, respond directly without tools.\n2. **Avoid redundant tool calls**: Do not call the same tool with the same arguments more than once in the same user turn. When an existing tool result already answers the user's request, stop calling tools and answer directly.\n3. **Use tools proactively**: When the user asks about files, time, system state, or needs actions performed, use the appropriate tool.\n4. **Chain tools when needed**: You can call multiple tools in sequence (e.g., list files → read file → summarize).\n5. **Handle failures gracefully**: If a tool fails, explain what went wrong and suggest alternatives. If it failed because of an unknown tool/platform/action name, retry once with an exact name from the returned suggestions, then explain rather than keep guessing.\n6. **Summarize results naturally**: After tool execution, synthesize the results into a helpful answer rather than dumping raw output.\n7. **Stay conversational**: Maintain a natural, helpful tone. Acknowledge context from earlier in the conversation.\n\n## Coding & Verification\nWhen working with code, prefer the precise built-in tools over ad-hoc shell: use `repo_map` to orient in an\nunfamiliar project, `code_search`/`file_find` to locate and `code_intel` for symbols, `edit_file` (anchored\nsearch-replace, or a unified `diff`) to change files — never rewrite a whole file to change a few lines — and\n`git_query` to inspect diffs/log. After edits, check `syntax_ok` in the result and run `test_run`/`lint_check`\nbefore declaring the work done. Batch independent read-only calls (search/read/query) into a single turn — they\nrun in parallel.\n\n## Presentation Style\n1. **Polished Markdown only**: Format user-facing answers with clean Markdown headings, short paragraphs, and concise bullets. Use tables only when they improve comparison or scanning.\n2. **Terminal-friendly layout**: Keep lines readable in a TUI; avoid dense walls of text, deeply nested lists, oversized ASCII art, or heavy visual blocks.\n3. **Elegant emphasis**: Use bold text sparingly for key terms and conclusions. Avoid excessive emojis, decorative symbols, repeated separators, or visual noise.\n4. **Theme-safe colors**: Do not emit ANSI escape codes, HTML color tags, Rich markup, or hardcoded color names. Rely on the TUI theme to render Markdown professionally.\n5. **No leaked tool protocol**: Never show tool-call JSON, internal schemas, raw observations, tool result payloads, or hidden reasoning in the final answer unless the user explicitly asks for raw/debug output. Treat any prior `{\"name\": ..., \"arguments\": ...}` blocks and `Tool result (...)` messages as internal execution context only.\n6. **Professional closure**: End with a concise conclusion or next step when helpful; avoid rambling after the useful answer is complete.\n\nWhen finished with all tool calls, respond normally without a JSON block, tool-call transcript, or process log.\n\n## Task Contract\n- Task ID: turn-1\n- Original user request: Let me show you something.\n- Workspace root: \n- Allowed roots: \n- Treat relative project paths as relative to the workspace root; never infer `.` as the project root when a workspace root is provided.\n- Workspace boundary is enforced by tools: do not read, search, edit, or run commands against paths outside the allowed roots unless the user explicitly requests an external path and the tool/approval policy permits it.\n- LeapFlow workspace config is optional at `/.leapflow/config.yaml`; runtime config is loaded from `~/.leapflow/config/user.yaml` and `~/.leapflow/profiles//config/*.yaml`.\n- Preserve this task contract across summarization, compression, tool loops, and memory retrieval.\n" + }, + { + "role": "user", + "content": "Let me show you something.\nLet me show you something." + } + ], + "tools": [ + "capability_expand", + "code_intel", + "code_search", + "config_get", + "config_list", + "env_info", + "file_find", + "file_list", + "file_read", + "git_query", + "lint_check", + "memory_search", + "repo_map", + "research_note", + "schedule_reentry", + "skill_view", + "skills_list", + "terminal_list", + "terminal_read", + "test_run", + "text_search", + "time_get", + "web_fetch" + ] + }, + "responses": [ + { + "status": 200, + "content_type": "application/json", + "body": "{\"model\":\"qwen3.7-plus\",\"id\":\"chatcmpl-04554729-8619-9387-95f5-8e3b16338531\",\"choices\":[{\"message\":{\"content\":\"Of course! I'm all ears — go ahead and share what you'd like to show me.\",\"reasoning_content\":\"The user says \\\"Let me show you something.\\\" This is a conversational opener - they want to show me something but haven't shared it yet. I should respond naturally and invite them to share what they'd like to show me. No tools needed here.\",\"role\":\"assistant\"},\"index\":0,\"finish_reason\":\"stop\"}],\"created\":1786001276,\"object\":\"chat.completion\",\"usage\":{\"total_tokens\":8777,\"completion_tokens\":77,\"prompt_tokens\":8700,\"completion_tokens_details\":{\"reasoning_tokens\":52,\"text_tokens\":77},\"prompt_tokens_details\":{\"cached_tokens\":0,\"text_tokens\":8700}}}" + } + ] +} diff --git a/tests/_harness/__init__.py b/tests/_harness/__init__.py new file mode 100644 index 0000000..001124d --- /dev/null +++ b/tests/_harness/__init__.py @@ -0,0 +1,82 @@ +"""Test harness for the real end-to-end layer. + +Modules here are infrastructure, not tests: + +``cassette`` + Fingerprinting, persistence and miss diagnostics for recorded LLM traffic. +``cassette_proxy`` + A local OpenAI-compatible endpoint that records, replays or forwards. +``leapd`` + Spawns and drives a real daemon subprocess. +``journey`` + The coarse-grained journey runner, with phase attribution and budgets. +""" + +from __future__ import annotations + +from tests._harness.cassette import ( + CassetteRecord, + CassetteResponse, + CassetteStore, + context_overflow_response, + error_response, + fingerprint, + json_response, + rate_limited_response, + record_for, + server_error_response, + streamed_response, + truncated_stream_response, +) +from tests._harness.cassette_proxy import ( + LIVE, + MODE_ENV, + RECORD, + REPLAY, + SEED, + CassetteProxy, + Script, + ScriptedTurn, + answer, + resolve_mode, + scripted, + store_for, + tool_call, +) +from tests._harness.journey import Journey, JourneyFactory, JourneyPhaseError +from tests._harness.leapd import Leapd, await_for, hermetic_env, start_leapd + +__all__ = [ + "LIVE", + "MODE_ENV", + "RECORD", + "REPLAY", + "SEED", + "CassetteProxy", + "CassetteRecord", + "CassetteResponse", + "CassetteStore", + "Journey", + "JourneyFactory", + "JourneyPhaseError", + "Leapd", + "Script", + "ScriptedTurn", + "answer", + "await_for", + "context_overflow_response", + "error_response", + "fingerprint", + "hermetic_env", + "json_response", + "rate_limited_response", + "record_for", + "resolve_mode", + "scripted", + "server_error_response", + "start_leapd", + "store_for", + "streamed_response", + "tool_call", + "truncated_stream_response", +] diff --git a/tests/_harness/cassette.py b/tests/_harness/cassette.py new file mode 100644 index 0000000..3419b87 --- /dev/null +++ b/tests/_harness/cassette.py @@ -0,0 +1,523 @@ +"""Cassette store: request fingerprinting, persistence, and miss diagnostics. + +A cassette is one recorded OpenAI-compatible HTTP exchange. Recording real +provider traffic — instead of hand-writing response bodies — is what keeps the +LLM boundary honest: the real ``openai`` SDK, the real ``httpx`` stack and the +real SSE framing all stay in the path, so a provider-parsing defect surfaces in +a test rather than in production. + +Design notes: + +- **Fingerprints normalize away volatile content.** A prompt embeds timestamps, + session ids and temp paths that change every run; without scrubbing them no + cassette would ever match twice. +- **One cassette holds a *sequence* of responses.** Retry and failover paths + send the *same* request repeatedly and must see *different* answers (429 then + 200). Responses are consumed in order and the last one repeats. +- **Failure injection is just an authored cassette.** A 429, a 500, a context + overflow and a truncated SSE stream are all ordinary recordings, so replay + needs no special-case branch and the stored file still looks like real wire + traffic. +""" + +from __future__ import annotations + +import base64 +import difflib +import hashlib +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +CASSETTE_SUFFIX = ".cassette.json" + +# Volatile substrings that must not enter a fingerprint. Ordered: broader +# patterns last, so an ISO timestamp is not first eaten by the digit rule. +_SCRUBBERS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?"), ""), + (re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"), ""), + # Undashed hex identifiers. Tool results carry a fresh ``execution_id`` on + # every call, so without this a single tool use makes the whole prompt + # unfingerprintable and no tool-using journey could ever replay. Bounded at 24 + # chars so ordinary hex-looking content (short hashes, colours) is untouched. + (re.compile(r"\b[0-9a-f]{24,}\b"), ""), + (re.compile(r"\b(?:sess|ws|req|traj|ep|skill|watch|call)-[0-9a-zA-Z]{6,}\b"), ""), + (re.compile(r"\bcall_[0-9a-zA-Z]{6,}\b"), ""), + (re.compile(r"/(?:private/)?(?:var|tmp)/[^\s\"',)\]]*"), ""), + (re.compile(r"127\.0\.0\.1:\d+"), "127.0.0.1:"), + (re.compile(r"\b1[0-9]{9}(?:\.[0-9]+)?\b"), ""), +) + + +def scrub(text: str) -> str: + """Replace run-varying substrings so equivalent prompts fingerprint equally.""" + for pattern, replacement in _SCRUBBERS: + text = pattern.sub(replacement, text) + return text + + +def _normalize_content(content: Any) -> Any: + """Normalize a message ``content`` field, which may be text or multimodal.""" + if isinstance(content, str): + return scrub(content) + if isinstance(content, list): + parts: list[Any] = [] + for part in content: + if isinstance(part, Mapping): + kind = part.get("type", "") + if kind == "text": + parts.append({"type": "text", "text": scrub(str(part.get("text", "")))}) + else: + # Image/audio payloads are large and byte-unstable; their + # presence and kind is what shapes the request. + parts.append({"type": kind}) + else: + parts.append(scrub(str(part))) + return parts + if content is None: + return None + return scrub(str(content)) + + +def normalize_request(payload: Mapping[str, Any]) -> dict[str, Any]: + """Reduce a chat-completions body to its fingerprint-relevant shape. + + Tools are reduced to their *names*: including full JSON schemas would + invalidate every cassette on a one-word description tweak, which is the + fastest way to make a replay suite unusable. + """ + messages: list[dict[str, Any]] = [] + for message in payload.get("messages") or []: + if not isinstance(message, Mapping): + continue + entry: dict[str, Any] = { + "role": str(message.get("role", "")), + "content": _normalize_content(message.get("content")), + } + calls = message.get("tool_calls") or [] + if calls: + entry["tool_calls"] = [ + str((call.get("function") or {}).get("name", "")) + for call in calls + if isinstance(call, Mapping) + ] + if message.get("tool_call_id"): + entry["tool_result"] = True + messages.append(entry) + + tool_names = sorted( + str((tool.get("function") or {}).get("name", "")) + for tool in (payload.get("tools") or []) + if isinstance(tool, Mapping) + ) + + normalized: dict[str, Any] = { + "model": str(payload.get("model", "")), + "stream": bool(payload.get("stream", False)), + "messages": messages, + } + if tool_names: + normalized["tools"] = tool_names + temperature = payload.get("temperature") + if isinstance(temperature, (int, float)): + normalized["temperature"] = round(float(temperature), 2) + return normalized + + +def fingerprint(payload: Mapping[str, Any]) -> str: + """Return the stable cassette key for a chat-completions request body.""" + canonical = json.dumps(normalize_request(payload), sort_keys=True, ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +# ── Records ────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class CassetteResponse: + """One HTTP response, either a whole body or a sequence of SSE frames.""" + + status: int = 200 + body: bytes = b"" + frames: tuple[bytes, ...] = () + content_type: str = "application/json" + + @property + def is_stream(self) -> bool: + """True when this response replays as server-sent events.""" + return bool(self.frames) + + def to_json(self) -> dict[str, Any]: + """Serialize, preferring readable text over base64.""" + payload: dict[str, Any] = {"status": self.status, "content_type": self.content_type} + if self.frames: + payload["frames"] = [_encode(frame) for frame in self.frames] + else: + payload["body"] = _encode(self.body) + return payload + + @classmethod + def from_json(cls, payload: Mapping[str, Any]) -> "CassetteResponse": + """Rebuild from :meth:`to_json` output.""" + frames = tuple(_decode(frame) for frame in payload.get("frames") or ()) + body = _decode(payload["body"]) if "body" in payload else b"" + return cls( + status=int(payload.get("status", 200)), + body=body, + frames=frames, + content_type=str(payload.get("content_type", "application/json")), + ) + + +@dataclass(frozen=True) +class CassetteRecord: + """A fingerprinted request paired with the responses it produced, in order.""" + + fingerprint: str + request: dict[str, Any] + responses: tuple[CassetteResponse, ...] + note: str = "" + + def to_json(self) -> dict[str, Any]: + """Serialize the whole record.""" + return { + "fingerprint": self.fingerprint, + "note": self.note, + "request": self.request, + "responses": [response.to_json() for response in self.responses], + } + + @classmethod + def from_json(cls, payload: Mapping[str, Any]) -> "CassetteRecord": + """Rebuild from :meth:`to_json` output.""" + return cls( + fingerprint=str(payload["fingerprint"]), + request=dict(payload.get("request") or {}), + responses=tuple( + CassetteResponse.from_json(item) for item in payload.get("responses") or () + ), + note=str(payload.get("note", "")), + ) + + def appended(self, response: CassetteResponse) -> "CassetteRecord": + """Return a copy with one more response at the end of the sequence.""" + return CassetteRecord( + fingerprint=self.fingerprint, + request=self.request, + responses=self.responses + (response,), + note=self.note, + ) + + +def _encode(raw: bytes) -> str | dict[str, str]: + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return {"b64": base64.b64encode(raw).decode("ascii")} + + +def _decode(value: Any) -> bytes: + if isinstance(value, Mapping): + return base64.b64decode(value["b64"]) + return str(value).encode("utf-8") + + +def total_tokens_of(response: "CassetteResponse") -> int: + """Return the provider-reported total token count for one response. + + Read from the response itself rather than from LeapFlow's own bookkeeping, so + the number is the provider's and covers every client (primary, aux, VLM) + without depending on which of them recorded what. Streamed answers report + usage on a late frame, so frames are scanned newest-first. + """ + if response.frames: + for frame in reversed(response.frames): + for payload in _json_objects_in(frame): + usage = payload.get("usage") + if isinstance(usage, Mapping): + total = usage.get("total_tokens") + if isinstance(total, int): + return total + return 0 + if not response.body: + return 0 + try: + payload = json.loads(response.body.decode("utf-8", errors="replace")) + except json.JSONDecodeError: + return 0 + usage = payload.get("usage") if isinstance(payload, Mapping) else None + if isinstance(usage, Mapping) and isinstance(usage.get("total_tokens"), int): + return int(usage["total_tokens"]) + return 0 + + +def _json_objects_in(frame: bytes) -> Iterable[dict[str, Any]]: + """Yield the JSON objects carried by the ``data:`` lines of an SSE frame.""" + for line in frame.decode("utf-8", errors="replace").splitlines(): + line = line.strip() + if not line.startswith("data:"): + continue + body = line[len("data:") :].strip() + if not body or body == "[DONE]": + continue + try: + parsed = json.loads(body) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + yield parsed + + +# ── Store ──────────────────────────────────────────────────────────────── + + +class CassetteMiss(LookupError): + """Raised in replay mode when no cassette matches the incoming request.""" + + +@dataclass +class CassetteStore: + """Directory-backed cassette collection, indexed by fingerprint.""" + + root: Path + _records: dict[str, CassetteRecord] = field(default_factory=dict) + _paths: dict[str, Path] = field(default_factory=dict) + + def __post_init__(self) -> None: + self.reload() + + def reload(self) -> None: + """Re-read every cassette file under ``root``.""" + self._records.clear() + self._paths.clear() + if not self.root.is_dir(): + return + for path in sorted(self.root.rglob(f"*{CASSETTE_SUFFIX}")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"unreadable cassette {path}: {exc}") from exc + record = CassetteRecord.from_json(payload) + self._records[record.fingerprint] = record + self._paths[record.fingerprint] = path + + def __len__(self) -> int: + return len(self._records) + + def keys(self) -> Iterable[str]: + """Return every known fingerprint.""" + return tuple(self._records) + + def get(self, key: str) -> CassetteRecord | None: + """Return the record for ``key``, or None when absent.""" + return self._records.get(key) + + def put(self, record: CassetteRecord) -> Path: + """Persist ``record``, overwriting any earlier version.""" + path = self._paths.get(record.fingerprint) or (self.root / self._filename(record)) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(record.to_json(), indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + self._records[record.fingerprint] = record + self._paths[record.fingerprint] = path + return path + + @staticmethod + def _filename(record: CassetteRecord) -> str: + model = re.sub(r"[^a-zA-Z0-9._-]+", "-", str(record.request.get("model", "model"))) + return f"{model}-{record.fingerprint[:16]}{CASSETTE_SUFFIX}" + + def explain_miss(self, payload: Mapping[str, Any]) -> str: + """Describe a miss by diffing against the closest stored request. + + A bare "no cassette for " is unusable: every prompt edit produces + one and gives no hint which message drifted, so the only response is to + re-record everything. The nearest-neighbour diff names the change. + """ + wanted = normalize_request(payload) + rendered = json.dumps(wanted, indent=2, sort_keys=True, ensure_ascii=False) + header = ( + f"no cassette for fingerprint {fingerprint(payload)} " + f"(model={wanted.get('model')}, stream={wanted.get('stream')}, " + f"{len(wanted.get('messages') or [])} messages, {len(self._records)} cassettes loaded)" + ) + if not self._records: + return f"{header}\nStore {self.root} is empty — run `make seed-cassettes`." + + best_key, best_ratio = "", -1.0 + for key, record in self._records.items(): + candidate = json.dumps(record.request, indent=2, sort_keys=True, ensure_ascii=False) + ratio = difflib.SequenceMatcher(None, rendered, candidate).quick_ratio() + if ratio > best_ratio: + best_key, best_ratio = key, ratio + + nearest = json.dumps( + self._records[best_key].request, indent=2, sort_keys=True, ensure_ascii=False + ) + diff = difflib.unified_diff( + nearest.splitlines(), + rendered.splitlines(), + fromfile=f"nearest cassette {self._paths[best_key].name}", + tofile="incoming request", + lineterm="", + n=2, + ) + return ( + f"{header}\nNearest stored request (similarity {best_ratio:.0%}):\n" + + "\n".join(list(diff)[:80]) + ) + + +# ── Authoring helpers (failure injection) ──────────────────────────────── + + +def sse_frames(*deltas: str, finish_reason: str = "stop", model: str = "cassette") -> tuple[bytes, ...]: + """Build SSE frames for a streamed text answer, terminated with ``[DONE]``.""" + frames: list[bytes] = [] + for delta in deltas: + chunk = { + "id": "chatcmpl-cassette", + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {"content": delta}, "finish_reason": None}], + } + frames.append(f"data: {json.dumps(chunk)}\n\n".encode("utf-8")) + tail = { + "id": "chatcmpl-cassette", + "object": "chat.completion.chunk", + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 64, "completion_tokens": 16, "total_tokens": 80}, + } + frames.append(f"data: {json.dumps(tail)}\n\n".encode("utf-8")) + frames.append(b"data: [DONE]\n\n") + return tuple(frames) + + +def streamed_response(*deltas: str, model: str = "cassette") -> CassetteResponse: + """A normal streamed 200 response carrying ``deltas``.""" + return CassetteResponse( + status=200, + frames=sse_frames(*deltas, model=model), + content_type="text/event-stream", + ) + + +def json_response( + *, + content: str = "", + tool_calls: Sequence[Mapping[str, Any]] = (), + model: str = "cassette", + finish_reason: str = "", +) -> CassetteResponse: + """A non-streamed 200 chat completion, optionally carrying native tool calls. + + The engine's native-tool round calls ``achat(stream=False)``, so tool-calling + turns are whole-body JSON rather than SSE. Journeys therefore need both wire + forms, and which one applies is decided by the request, not by the author. + """ + message: dict[str, Any] = {"role": "assistant", "content": content} + calls: list[dict[str, Any]] = [] + for index, call in enumerate(tool_calls): + arguments = call.get("arguments", {}) + calls.append( + { + "id": str(call.get("id") or f"call_{index + 1}"), + "type": "function", + "function": { + "name": str(call.get("name", "")), + "arguments": arguments + if isinstance(arguments, str) + else json.dumps(arguments, ensure_ascii=False), + }, + } + ) + if calls: + message["tool_calls"] = calls + body = { + "id": "chatcmpl-cassette", + "object": "chat.completion", + "model": model, + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": finish_reason or ("tool_calls" if calls else "stop"), + } + ], + "usage": {"prompt_tokens": 64, "completion_tokens": 16, "total_tokens": 80}, + } + return CassetteResponse(status=200, body=json.dumps(body).encode("utf-8")) + + +def truncated_stream_response(*deltas: str, model: str = "cassette") -> CassetteResponse: + """A streamed response that stops mid-flight, with no terminating frame. + + Models a provider dropping the connection: the client must recover rather + than hand a half-parsed answer to the user. + """ + frames = sse_frames(*deltas, model=model) + return CassetteResponse( + status=200, + frames=frames[: max(1, len(frames) - 2)], + content_type="text/event-stream", + ) + + +def error_response(status: int, *, code: str, message: str, kind: str = "invalid_request_error") -> CassetteResponse: + """An OpenAI-shaped error body at ``status``. + + The shape matters: the recovery classifier reads provider error payloads, so + an injected failure must look exactly like the real thing. + """ + body = json.dumps({"error": {"message": message, "type": kind, "code": code}}) + return CassetteResponse(status=status, body=body.encode("utf-8")) + + +def rate_limited_response() -> CassetteResponse: + """A 429 the provider layer is expected to retry.""" + return error_response( + 429, + code="rate_limit_exceeded", + message="Rate limit reached for requests", + kind="rate_limit_error", + ) + + +def server_error_response() -> CassetteResponse: + """A 500 the provider layer is expected to retry.""" + return error_response( + 500, code="internal_error", message="The server had an error", kind="server_error" + ) + + +def context_overflow_response(*, limit: int = 8192, requested: int = 9001) -> CassetteResponse: + """A 400 context-length error, the trigger for context compression.""" + return error_response( + 400, + code="context_length_exceeded", + message=( + f"This model's maximum context length is {limit} tokens. " + f"However, your messages resulted in {requested} tokens." + ), + ) + + +def record_for( + payload: Mapping[str, Any], + *responses: CassetteResponse, + note: str = "", +) -> CassetteRecord: + """Author a cassette for ``payload`` with an explicit response sequence.""" + if not responses: + raise ValueError("a cassette needs at least one response") + return CassetteRecord( + fingerprint=fingerprint(payload), + request=normalize_request(payload), + responses=tuple(responses), + note=note, + ) diff --git a/tests/_harness/cassette_proxy.py b/tests/_harness/cassette_proxy.py new file mode 100644 index 0000000..47a8ecd --- /dev/null +++ b/tests/_harness/cassette_proxy.py @@ -0,0 +1,611 @@ +"""Local OpenAI-compatible proxy that records, replays, or forwards LLM traffic. + +Why a proxy instead of patching the provider: ``OpenAIChat`` builds its +``AsyncOpenAI`` client internally, and leapd runs as a *separate process* +(``sys.executable -m leapflow``), so in-process patching cannot reach it. A +proxy is addressed the way production addresses any provider — through +``LEAPFLOW_LLM_BASE_URL`` — which means the real ``openai`` SDK, the real +``httpx`` stack, the real SSE framing and the real retry classification all stay +in the path. + +Modes (``LEAPFLOW_TEST_LLM_MODE``): + +``replay`` + Serve from the cassette store. A miss is a hard failure carrying a + nearest-neighbour diff, never a silent fallthrough. +``seed`` + Serve from the journey's declared script and persist each exchange as a + cassette. This bootstraps a committed, offline-runnable store before any + real credential exists; ``record`` later replaces those bodies with real + ones. +``record`` + Forward to the real upstream and persist what comes back — into a *separate* + ``recordings/`` store. It is evidence of what providers really send, not a + replay input: a multi-turn agent conversation cannot be replayed from a + recording, because turn *n*'s prompt embeds the exact round-by-round history + of every turn before it. +``live`` + Forward to the real upstream and persist nothing. +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping + +import httpx + +from tests._harness.cassette import ( + CassetteRecord, + CassetteResponse, + CassetteStore, + fingerprint, + json_response, + normalize_request, + streamed_response, + total_tokens_of, +) + +logger = logging.getLogger(__name__) + +MODE_ENV = "LEAPFLOW_TEST_LLM_MODE" +REPLAY = "replay" +SEED = "seed" +RECORD = "record" +LIVE = "live" +_MODES = (REPLAY, SEED, RECORD, LIVE) + +_FORWARD_MODES = (RECORD, LIVE) + + +def resolve_mode(default: str = REPLAY) -> str: + """Return the configured proxy mode, validating it early.""" + mode = (os.getenv(MODE_ENV, "") or default).strip().lower() + if mode not in _MODES: + raise ValueError(f"{MODE_ENV}={mode!r} is not one of {_MODES}") + return mode + + +@dataclass(frozen=True) +class ScriptedTurn: + """One model turn expressed as *semantics*, rendered to fit the request. + + The engine picks the wire form: a native-tool round sends ``stream=false`` + and expects a whole JSON body, while a plain answer round streams SSE. + Declaring "answer this" or "call that tool" and rendering on demand keeps + journeys readable and stops them from encoding a transport detail they do + not control. + """ + + text: str = "" + tool_calls: tuple[Mapping[str, Any], ...] = () + + def render(self, *, stream: bool, model: str) -> CassetteResponse: + """Return the response body appropriate for this request shape.""" + if self.tool_calls or not stream: + return json_response( + content=self.text, tool_calls=self.tool_calls, model=model + ) + return streamed_response(*_split_for_streaming(self.text), model=model) + + +def answer(text: str) -> ScriptedTurn: + """Script a final textual answer.""" + return ScriptedTurn(text=text) + + +def tool_call(name: str, **arguments: Any) -> ScriptedTurn: + """Script a single native tool call.""" + return ScriptedTurn(tool_calls=({"name": name, "arguments": arguments},)) + + +def _split_for_streaming(text: str, *, parts: int = 3) -> tuple[str, ...]: + """Split an answer into a few deltas so multi-frame SSE parsing is exercised.""" + if not text: + return ("",) + size = max(1, -(-len(text) // parts)) + return tuple(text[index : index + size] for index in range(0, len(text), size)) + + +@dataclass +class Script: + """Ordered turns served to requests that have no cassette yet. + + A script is a *seed*, not an assertion target: it exists so a journey can be + written and committed before a live credential is available. Its bodies are + superseded by real recordings in ``record`` mode, and the shape check in + ``tools/sync_fixtures.py`` is what keeps a seeded body from drifting away + from what providers actually send. + + The **last turn repeats** for every call past the end of the script, so it + must be a benign final answer. Ending on a tool call or a structured payload + makes the agent loop keep re-reading it and burn its whole iteration budget. + """ + + turns: list[ScriptedTurn | CassetteResponse] = field(default_factory=list) + _index: int = 0 + + @classmethod + def of(cls, *entries: str | ScriptedTurn | CassetteResponse) -> "Script": + """Build a script from answer texts, scripted turns, or raw responses.""" + built: list[ScriptedTurn | CassetteResponse] = [] + for entry in entries: + if isinstance(entry, (ScriptedTurn, CassetteResponse)): + built.append(entry) + else: + built.append(ScriptedTurn(text=str(entry))) + return cls(turns=built) + + def next_response(self, *, stream: bool, model: str) -> CassetteResponse | None: + """Return the next scripted response; the last turn repeats.""" + if not self.turns: + return None + entry = self.turns[min(self._index, len(self.turns) - 1)] + self._index += 1 + if isinstance(entry, CassetteResponse): + return entry + return entry.render(stream=stream, model=model) + + +@dataclass +class ProxyStats: + """Observable traffic for journey assertions.""" + + requests: list[dict[str, Any]] = field(default_factory=list) + misses: list[str] = field(default_factory=list) + upstream_calls: int = 0 + total_tokens: int = 0 + budget_exceeded: bool = False + token_budget_exceeded: bool = False + + @property + def call_count(self) -> int: + """Number of chat-completions requests the proxy handled.""" + return len(self.requests) + + def prompts_containing(self, needle: str) -> list[dict[str, Any]]: + """Return requests whose normalized messages mention ``needle``.""" + found = [] + for request in self.requests: + blob = json.dumps(request.get("messages") or [], ensure_ascii=False) + if needle in blob: + found.append(request) + return found + + +class CassetteProxy: + """An OpenAI-compatible HTTP endpoint backed by a cassette store.""" + + def __init__( + self, + store: CassetteStore, + *, + mode: str = REPLAY, + script: Script | None = None, + upstream_base_url: str = "", + upstream_api_key: str = "", + upstream_model: str = "", + host: str = "127.0.0.1", + max_calls: int = 0, + max_tokens: int = 0, + ) -> None: + if mode not in _MODES: + raise ValueError(f"unknown mode {mode!r}") + self._store = store + self._mode = mode + self._script = script or Script() + self._upstream_base_url = upstream_base_url.rstrip("/") + self._upstream_api_key = upstream_api_key + self._upstream_model = upstream_model.strip() + self._host = host + self._max_calls = max(0, int(max_calls)) + self._max_tokens = max(0, int(max_tokens)) + self._lock = threading.Lock() + self._cursor: dict[str, int] = {} + self._captured: set[str] = set() + self._server: ThreadingHTTPServer | None = None + self._thread: threading.Thread | None = None + self.stats = ProxyStats() + + if mode in _FORWARD_MODES and not self._upstream_base_url: + raise ValueError(f"mode {mode!r} needs an upstream base URL (LEAPFLOW_LLM_BASE_URL)") + + # ── Lifecycle ──────────────────────────────────────────────────── + + @property + def mode(self) -> str: + """Configured proxy mode.""" + return self._mode + + @property + def base_url(self) -> str: + """OpenAI-compatible base URL to hand to LeapFlow via config/env.""" + if self._server is None: + raise RuntimeError("proxy is not started") + host, port = self._server.server_address[:2] + return f"http://{host}:{port}/v1" + + def start(self) -> "CassetteProxy": + """Bind an ephemeral port and serve in a background thread.""" + proxy = self + + class _Handler(_CassetteHandler): + proxy_ref = proxy + + self._server = ThreadingHTTPServer((self._host, 0), _Handler) + self._server.daemon_threads = True + self._thread = threading.Thread( + target=self._server.serve_forever, name="cassette-proxy", daemon=True + ) + self._thread.start() + return self + + def stop(self) -> None: + """Shut the server down and join its thread.""" + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + if self._thread is not None: + self._thread.join(timeout=5.0) + self._thread = None + + def __enter__(self) -> "CassetteProxy": + return self.start() + + def __exit__(self, *exc: object) -> None: + self.stop() + + # ── Assertions ─────────────────────────────────────────────────── + + def assert_no_misses(self) -> None: + """Fail with full diagnostics when replay could not answer a request.""" + if self.stats.misses: + joined = "\n\n".join(self.stats.misses) + raise AssertionError( + f"{len(self.stats.misses)} cassette miss(es) in {self._mode} mode:\n\n{joined}" + ) + + # ── Request handling (called on server threads) ─────────────────── + + def handle_chat(self, payload: Mapping[str, Any]) -> CassetteResponse: + """Resolve one chat-completions request to a response. + + Mode decides precedence, and it matters for retry paths: a retry resends + a *byte-identical* request, so a recording mode must keep capturing + instead of answering from what it just stored — otherwise a "429 then + 200" sequence could never be recorded at all. + """ + key = fingerprint(payload) + stream = bool(payload.get("stream")) + model = str(payload.get("model") or "cassette") + with self._lock: + self.stats.requests.append(normalize_request(payload)) + over_calls = 0 < self._max_calls < len(self.stats.requests) + if over_calls: + self.stats.budget_exceeded = True + over_tokens = 0 < self._max_tokens <= self.stats.total_tokens + if over_tokens: + self.stats.token_budget_exceeded = True + if over_calls: + return self._refuse( + "journey_call_budget_exceeded", + f"journey exceeded its provider-call budget of {self._max_calls}. " + "Either the turn stopped converging, or the prompt grew enough to " + "need more rounds — both are regressions worth looking at, not a " + "reason to raise the ceiling.", + ) + if over_tokens: + return self._refuse( + "journey_token_budget_exceeded", + f"journey spent {self.stats.total_tokens} tokens, past its ceiling " + f"of {self._max_tokens}. Call count alone cannot catch this: prompt " + "assembly growing (a longer system prompt, more tool schemas) " + "raises the bill without adding a single round.", + ) + + response = self._resolve(key, payload, stream=stream, model=model) + with self._lock: + self.stats.total_tokens += total_tokens_of(response) + return response + + def _resolve( + self, key: str, payload: Mapping[str, Any], *, stream: bool, model: str + ) -> CassetteResponse: + """Answer one request from the mode's authoritative source.""" + if self._mode in _FORWARD_MODES: + response = self._forward(payload) + if self._mode == RECORD: + self._capture(key, payload, response) + return response + + if self._mode == SEED: + response = self._script.next_response(stream=stream, model=model) + if response is not None: + mismatch = _shape_mismatch(response, stream=stream) + if mismatch: + return self._author_error(mismatch) + self._capture(key, payload, response) + return response + + with self._lock: + record = self._store.get(key) + if record is not None: + index = self._cursor.get(key, 0) + self._cursor[key] = index + 1 + return record.responses[min(index, len(record.responses) - 1)] + + return self._miss(payload) + + def _refuse(self, code: str, message: str) -> CassetteResponse: + """Refuse further provider calls with a non-retryable status. + + Enforced here rather than only asserted afterwards, because the failure + being guarded against is a loop that does not converge: letting it run to + the engine's iteration cap costs minutes offline and real money live. A + 400 stops it at the ceiling — a 429 or 5xx would be retried and the loop + would continue. + """ + logger.error("cassette-proxy: %s", message) + body = json.dumps( + {"error": {"message": message, "type": "invalid_request_error", "code": code}} + ) + return CassetteResponse(status=400, body=body.encode("utf-8")) + + def _author_error(self, explanation: str) -> CassetteResponse: + """Report an authoring mistake as a failure of the test, not of the product. + + Serving an SSE body to a ``stream=false`` request makes the SDK parse a + string as a completion object; the resulting ``AttributeError`` is then + correctly classified as a LeapFlow defect, and the journey appears to have + found a product bug it did not find. Catching the mis-shape here keeps the + blame where it belongs. + """ + with self._lock: + self.stats.misses.append(explanation) + logger.error("cassette authoring error: %s", explanation) + body = json.dumps( + { + "error": { + "message": explanation, + "type": "invalid_request_error", + "code": "cassette_shape_mismatch", + } + } + ) + return CassetteResponse(status=400, body=body.encode("utf-8")) + + def _miss(self, payload: Mapping[str, Any]) -> CassetteResponse: + """Record a replay miss and answer with a non-retryable 400. + + 400 is deliberate: the provider retries 429/5xx, so answering a miss + with a 500 would burn the whole retry budget before the test could + report the real problem. + """ + explanation = self._store.explain_miss(payload) + with self._lock: + self.stats.misses.append(explanation) + logger.error("cassette miss: %s", explanation) + body = json.dumps( + { + "error": { + "message": f"cassette miss ({self._mode} mode): {explanation}", + "type": "invalid_request_error", + "code": "cassette_miss", + } + } + ) + return CassetteResponse(status=400, body=body.encode("utf-8")) + + def _capture( + self, key: str, payload: Mapping[str, Any], response: CassetteResponse + ) -> None: + """Store ``response`` for ``key``, replacing a stale run then appending. + + The first capture of a key in this proxy's lifetime *replaces* whatever a + previous run left behind; later captures of the same key append. Without + the replace, re-running the recorder would keep growing every sequence + and turn a one-off retry into a permanent one. + """ + with self._lock: + existing = self._store.get(key) if key in self._captured else None + if existing is not None: + record = existing.appended(response) + else: + record = CassetteRecord( + fingerprint=key, + request=normalize_request(payload), + responses=(response,), + note=f"captured in {self._mode} mode", + ) + self._captured.add(key) + self._store.put(record) + self._cursor[key] = len(record.responses) + + def _forward(self, payload: Mapping[str, Any]) -> CassetteResponse: + """Call the real upstream and capture its response verbatim. + + The outgoing ``model`` is rewritten to the provider's actual model while + the cassette keeps the journey's stable placeholder. Without that split, + recording under a real model name and replaying under the placeholder + produce different fingerprints, so every re-recorded cassette would miss + on the next replay run — which would break the record/replay cycle the + whole design rests on. + """ + url = f"{self._upstream_base_url}/chat/completions" + headers = { + "Authorization": f"Bearer {self._upstream_api_key}", + "Content-Type": "application/json", + } + outgoing = dict(payload) + if self._upstream_model: + outgoing["model"] = self._upstream_model + with self._lock: + self.stats.upstream_calls += 1 + timeout = httpx.Timeout(connect=30.0, read=180.0, write=30.0, pool=30.0) + if outgoing.get("stream"): + frames: list[bytes] = [] + with httpx.Client(timeout=timeout) as client: + with client.stream("POST", url, json=outgoing, headers=headers) as upstream: + status = upstream.status_code + content_type = upstream.headers.get("content-type", "text/event-stream") + if status >= 400: + return CassetteResponse( + status=status, + body=upstream.read(), + content_type=content_type, + ) + for raw in upstream.iter_raw(): + if raw: + frames.append(raw) + return CassetteResponse( + status=status, frames=tuple(frames), content_type=content_type + ) + with httpx.Client(timeout=timeout) as client: + upstream = client.post(url, json=outgoing, headers=headers) + return CassetteResponse( + status=upstream.status_code, + body=upstream.content, + content_type=upstream.headers.get("content-type", "application/json"), + ) + + +class _CassetteHandler(BaseHTTPRequestHandler): + """Minimal HTTP surface: chat completions plus a models probe.""" + + proxy_ref: CassetteProxy + protocol_version = "HTTP/1.1" + + def log_message(self, fmt: str, *args: Any) -> None: # noqa: A003 - stdlib hook + """Silence stdlib access logging; failures are reported by assertions.""" + logger.debug("cassette-proxy %s", fmt % args) + + def do_GET(self) -> None: + """Answer the model-listing probe some clients issue on startup.""" + if self.path.rstrip("/").endswith("/models"): + self._send(CassetteResponse(status=200, body=b'{"object":"list","data":[]}')) + return + self._send(CassetteResponse(status=404, body=b'{"error":{"message":"not found"}}')) + + def do_POST(self) -> None: + """Serve a chat-completions request from cassette, script, or upstream.""" + if not self.path.rstrip("/").endswith("/chat/completions"): + self._send(CassetteResponse(status=404, body=b'{"error":{"message":"not found"}}')) + return + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + self._send( + CassetteResponse( + status=400, + body=json.dumps({"error": {"message": f"bad request body: {exc}"}}).encode(), + ) + ) + return + response = self.proxy_ref.handle_chat(payload) + self._send(response) + + def _send(self, response: CassetteResponse) -> None: + """Write a whole body, or stream SSE frames with flushes between them.""" + if response.is_stream: + self.send_response(response.status) + self.send_header("Content-Type", response.content_type) + self.send_header("Cache-Control", "no-cache") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + try: + for frame in response.frames: + self.wfile.write(f"{len(frame):X}\r\n".encode("ascii")) + self.wfile.write(frame) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + logger.debug("cassette-proxy: client closed mid-stream") + return + self.send_response(response.status) + self.send_header("Content-Type", response.content_type) + self.send_header("Content-Length", str(len(response.body))) + self.end_headers() + try: + self.wfile.write(response.body) + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + logger.debug("cassette-proxy: client closed before body") + + +def store_for(journey_id: str, *, mode: str = REPLAY, root: Path | None = None) -> CassetteStore: + """Return the cassette store a journey should use in ``mode``. + + Recording writes to a **separate** directory from the replay store, and that + separation is load-bearing rather than tidiness: + + - A recording run must never be able to break the offline lanes. Writing real + traffic into the replay store does exactly that, because a multi-turn agent + conversation cannot be replayed from a recording: turn *n*'s prompt embeds + the round-by-round history of turns 1..n-1, so one divergence (a tool call + the model made this time but not last time) cascades and every later turn + misses. + - The two artefacts answer different questions. ``cassettes/`` holds the + deterministic inputs the replay lanes assert against; ``recordings/`` holds + evidence of what providers actually send, which ``tools/sync_fixtures.py`` + distils into the shapes the mock layer checks against. + + Journeys keep separate directories so a nearest-neighbour miss diff stays + relevant instead of matching an unrelated journey's prompt. + """ + base = root or Path(__file__).resolve().parents[1] / "_fixtures" + bucket = "recordings" if mode == RECORD else "cassettes" + return CassetteStore(base / bucket / journey_id) + + +def upstream_from_env() -> tuple[str, str, str]: + """Return (base_url, api_key, model) for forwarding modes, from process env.""" + base_url = os.getenv("LEAPFLOW_TEST_UPSTREAM_BASE_URL", "").strip() + api_key = os.getenv("LEAPFLOW_TEST_UPSTREAM_API_KEY", "").strip() + model = os.getenv("LEAPFLOW_TEST_UPSTREAM_MODEL", "").strip() + # The live lane injects the ordinary LLM variables; fall back to them so a + # single set of CI secrets drives both LeapFlow and the recorder. + if not base_url: + base_url = os.getenv("LEAPFLOW_LLM_BASE_URL", "").strip() + if not api_key: + api_key = os.getenv("LEAPFLOW_LLM_API_KEY", "").strip() + if not model: + model = os.getenv("LEAPFLOW_LLM_MODEL", "").strip() + return base_url, api_key, model + + +def _shape_mismatch(response: CassetteResponse, *, stream: bool) -> str: + """Return an explanation when a response cannot answer this request shape. + + Only successful bodies are checked: an error status is shape-neutral, and + providers really do answer a streaming request with a plain JSON error. + """ + if response.status >= 400: + return "" + if response.is_stream and not stream: + return ( + "scripted an SSE response for a stream=false request. The engine's " + "native-tool round is non-streaming, so use answer()/tool_call() — they " + "render to fit the request — rather than a raw streamed_response()." + ) + if stream and not response.is_stream: + return ( + "scripted a whole-body response for a stream=true request. Use " + "answer()/tool_call() so the wire form follows the request." + ) + return "" + + +def scripted(*entries: str | ScriptedTurn | CassetteResponse) -> Script: + """Shorthand for :meth:`Script.of`.""" + return Script.of(*entries) diff --git a/tests/_harness/journey.py b/tests/_harness/journey.py new file mode 100644 index 0000000..2255a92 --- /dev/null +++ b/tests/_harness/journey.py @@ -0,0 +1,259 @@ +"""Journey runner: one coarse end-to-end test made diagnosable. + +The real layer is deliberately small — a handful of journeys, each covering many +user-facing features as ordered phases inside a single session. Coarse tests buy +cross-module coverage at the cost of failure localization, so :meth:`Journey.phase` +buys the localization back: a failure names the phase it happened in and carries +the daemon's own log tail, which is the only place a cross-process cause is +recorded. +""" + +from __future__ import annotations + +import contextlib +import logging +import shutil +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterator + +import pytest + +from tests._harness.cassette_proxy import ( + LIVE, + RECORD, + CassetteProxy, + Script, + store_for, + upstream_from_env, +) +from tests._harness.leapd import Leapd, journey_root, start_leapd + +logger = logging.getLogger(__name__) + +DEFAULT_DEADLINE_S = 90.0 + +# A journey should never need many provider calls. The ceiling is what keeps a +# non-converging turn from burning the engine's whole iteration budget — offline +# that costs minutes, live it costs money. +DEFAULT_MAX_LLM_CALLS = 12 + +# Token ceiling, checked independently of the call count. Call count alone cannot +# catch prompt growth: a longer system prompt or a bigger tool catalogue raises +# the bill without adding a single round, which is precisely how a live lane's +# cost creeps up unnoticed. +DEFAULT_MAX_LLM_TOKENS = 150_000 + + +class JourneyPhaseError(AssertionError): + """A journey phase failed; carries the phase trail and daemon log tail.""" + + +@dataclass +class Journey: + """A running journey: cassette proxy, real leapd, and a phase trail.""" + + journey_id: str + proxy: CassetteProxy + daemon: Leapd + deadline_s: float = DEFAULT_DEADLINE_S + max_llm_calls: int = 0 + max_llm_tokens: int = 0 + started_at: float = field(default_factory=time.monotonic) + trail: list[str] = field(default_factory=list) + + @property + def elapsed_s(self) -> float: + """Seconds since the journey started.""" + return time.monotonic() - self.started_at + + @property + def is_live(self) -> bool: + """True when answers come from a real provider rather than a recording. + + Journeys use this to relax the assertions that depend on a *specific* + model choice ("it called this tool") while keeping the invariants that + must hold either way ("the turn produced an answer and no error"). The + strict form still runs on every push, in replay. + """ + return self.proxy.mode in (LIVE, RECORD) + + def client(self, *, timeout_s: float = 120.0) -> Any: + """Return a fresh RPC client for the journey's daemon.""" + return self.daemon.client(timeout_s=timeout_s) + + def workspace(self, name: str) -> Path: + """Create (once) and return an isolated workspace for this journey.""" + return self.daemon.workspace(name) + + @contextlib.contextmanager + def phase(self, label: str) -> Iterator[None]: + """Run one named stage, attributing any failure to it. + + Nested phases are supported and render as ``outer > inner``. + """ + self.trail.append(label) + crumb = " > ".join(self.trail) + began = time.monotonic() + logger.info("journey %s: phase %s", self.journey_id, crumb) + try: + yield + except JourneyPhaseError: + # An inner phase already attributed the failure; re-wrapping would bury + # the specific phase under the outer one. + raise + except Exception as exc: # re-raised with cross-process context attached + raise JourneyPhaseError( + f"journey {self.journey_id!r} failed in phase {crumb!r} " + f"after {self.elapsed_s:.1f}s: {type(exc).__name__}: {exc}\n" + f"--- leapd log tail ---\n{self.daemon.tail_log()}" + ) from exc + finally: + took = time.monotonic() - began + logger.info("journey %s: phase %s took %.2fs", self.journey_id, crumb, took) + self.trail.pop() + + def finish(self) -> None: + """Assert the journey's own contracts: no misses, within call and time budget. + + These budgets are part of the design, not a nicety. The real layer earns + the right to run on every push only by staying cheap, and a turn that + stops converging shows up here first — as more provider calls than the + journey should ever need. + """ + self.proxy.assert_no_misses() + if self.proxy.stats.budget_exceeded: + raise AssertionError( + f"journey {self.journey_id!r} exhausted its provider-call budget of " + f"{self.max_llm_calls} after {self.proxy.stats.call_count} calls. " + "A turn stopped converging, or prompt assembly grew enough to need " + "extra rounds; investigate rather than raising the ceiling." + ) + if self.proxy.stats.token_budget_exceeded: + raise AssertionError( + f"journey {self.journey_id!r} spent {self.proxy.stats.total_tokens} " + f"tokens, past its ceiling of {self.max_llm_tokens}. The call count " + "stayed within budget, so this is prompt growth, not a loop — and it " + "is what would otherwise raise the live lane's bill in silence." + ) + logger.info( + "journey %s: %d provider call(s), %d token(s), %.1fs", + self.journey_id, + self.proxy.stats.call_count, + self.proxy.stats.total_tokens, + self.elapsed_s, + ) + if self.elapsed_s > self.deadline_s: + raise AssertionError( + f"journey {self.journey_id!r} took {self.elapsed_s:.1f}s, " + f"over its {self.deadline_s:.0f}s budget — split a phase out or " + f"reduce the number of turns rather than raising the budget" + ) + + +class JourneyFactory: + """Builds journeys and owns their teardown for one test. + + Each journey gets a clean, deterministic scratch root rather than pytest's + ``tmp_path``: a daemon needs a Unix socket short enough for the kernel, and + recorded tool calls embed absolute paths that only resolve if the workspace + lands in the same place on every run. + """ + + def __init__(self, mode: str) -> None: + self._mode = mode + self._stack = contextlib.ExitStack() + self._journeys: list[Journey] = [] + self._roots: list[Path] = [] + + def __call__( + self, + journey_id: str, + *, + script: Script | None = None, + deadline_s: float = DEFAULT_DEADLINE_S, + max_llm_calls: int = DEFAULT_MAX_LLM_CALLS, + max_llm_tokens: int = DEFAULT_MAX_LLM_TOKENS, + requires_scripted_responses: bool = False, + extra_env: dict[str, str] | None = None, + model: str = "cassette-model", + profile: str = "default", + ) -> Journey: + """Start a cassette proxy and a real leapd, returning the journey handle. + + Args: + journey_id: Stable id; also names the journey's cassette directory. + script: Responses used to seed cassettes when none exist yet. + deadline_s: Wall-clock ceiling asserted by :meth:`Journey.finish`. + max_llm_calls: Provider-call ceiling. Enforced by the proxy, so a + turn that stops converging is cut off instead of running to the + engine's iteration cap. + max_llm_tokens: Token ceiling, enforced independently. Call count + cannot catch prompt growth, which raises cost without adding a + round. + requires_scripted_responses: Set when the journey's assertions depend + on responses only a recording can produce — injected 429s, a + context overflow, a specific tool call. Forwarding modes cannot + produce those, so the journey skips rather than failing for a + reason that has nothing to do with the product. + extra_env: Additional ``LEAPFLOW_*`` overrides. + model: Model name recorded in cassette fingerprints. + profile: Profile id to create and activate. + """ + if requires_scripted_responses and self._mode in (LIVE, RECORD): + pytest.skip( + f"journey {journey_id!r} asserts on injected provider behavior, which " + f"{self._mode!r} mode cannot produce — it forwards every request " + "upstream. This journey is meaningful in replay only." + ) + + upstream_base_url, upstream_api_key, upstream_model = upstream_from_env() + proxy = CassetteProxy( + store_for(journey_id, mode=self._mode), + mode=self._mode, + script=script, + upstream_base_url=upstream_base_url, + upstream_api_key=upstream_api_key, + upstream_model=upstream_model, + max_calls=max_llm_calls, + max_tokens=max_llm_tokens, + ) + self._stack.enter_context(proxy) + + # The daemon always sees the journey's stable placeholder model, in every + # mode. Cassette fingerprints include the model, so letting the real + # provider's name reach the daemon would make recordings unreplayable; the + # proxy rewrites the name on the wire instead. + root = journey_root(journey_id) + self._roots.append(root) + daemon = start_leapd( + root=root, + llm_base_url=proxy.base_url, + llm_model=model, + profile=profile, + extra_env=extra_env, + ) + self._stack.callback(daemon.stop) + + journey = Journey( + journey_id=journey_id, + proxy=proxy, + daemon=daemon, + deadline_s=deadline_s, + max_llm_calls=max_llm_calls, + max_llm_tokens=max_llm_tokens, + ) + self._journeys.append(journey) + return journey + + def close(self) -> None: + """Tear down every journey started through this factory. + + Scratch roots are left in place after teardown of the *processes* only + long enough to be removed here; the next run also clears them, so a + crashed run cannot leak state into the next one either way. + """ + self._stack.close() + for root in self._roots: + shutil.rmtree(root, ignore_errors=True) diff --git a/tests/_harness/leapd.py b/tests/_harness/leapd.py new file mode 100644 index 0000000..bcc1345 --- /dev/null +++ b/tests/_harness/leapd.py @@ -0,0 +1,324 @@ +"""Spawn and drive a real ``leapd`` subprocess for end-to-end journeys. + +Journeys talk to the daemon over its actual Unix-socket RPC, because that is the +only place a whole class of defects is observable: engine template vs. per-session +instance, session identity, workspace binding, cross-process persistence and +pushed runtime metadata all look correct inside one process and break across +two. In-process fixtures cannot see any of it. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + +from leapflow.daemon.client import DaemonClient +from leapflow.daemon.lifecycle import DaemonInfo, cleanup_stale, wait_ready +from leapflow.layout import build_layout + +READY_TIMEOUT_S = 60.0 +STOP_TIMEOUT_S = 15.0 + +# A Unix socket path is length-limited by the kernel (104 bytes on macOS, 108 on +# Linux). pytest's tmp_path is deep enough on macOS to blow through it, and the +# resulting failure is a bare OSError from inside the daemon — invisible in the +# test process. Journeys therefore run from a short root, and this bound is +# checked before the process is spawned so the message names the real cause. +MAX_SOCKET_PATH_LEN = 100 + + +def journey_root(journey_id: str, *, prefix: str = "lfj-") -> Path: + """Return a clean, *deterministic* scratch root for one journey. + + Deterministic rather than random for two reasons, both learned the hard way: + + - A Unix socket path is length-limited by the kernel (104 bytes on macOS). + pytest's ``tmp_path`` is deep enough to exceed it, and the failure is a bare + ``OSError`` inside the daemon that the test process never sees. + - Recorded tool calls embed **absolute** paths chosen by the model. Replaying + them under a fresh random directory puts those paths outside the new + workspace, the tool refuses with ``outside_workspace``, and the resulting + tool result no longer matches what was recorded — so every tool-using + journey misses its cassette. A stable path makes the recording replayable. + + The directory is removed first, so each run starts from empty state and a + crashed previous run cannot leak a session or a database into this one. + + Raises: + ConcurrentJourneyError: Another run of *this* journey is live in the same + directory. Because the path is deterministic, two concurrent runs of + one journey would share it; refusing is far better than deleting the + other run's daemon state and leaving both to fail confusingly. + """ + base = Path("/tmp") + if not (base.is_dir() and os.access(base, os.W_OK)): + base = Path(tempfile.gettempdir()) + root = base / f"{prefix}{journey_id}" + + live = _live_daemon_in(root) + if live is not None: + raise ConcurrentJourneyError( + f"journey {journey_id!r} already has a live daemon (pid {live}) under " + f"{root}. Its scratch root is deterministic — recorded tool calls embed " + "absolute paths, so it has to be — which means two runs of the same " + "journey cannot share a machine. Wait for the other run, or stop that " + "daemon before retrying." + ) + + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True, exist_ok=True) + return root + + +def _live_daemon_in(root: Path) -> int | None: + """Return the pid of a healthy daemon under ``root``, or None.""" + for runtime_dir in root.glob("data/profiles/*/runtime"): + info = DaemonInfo.discover(runtime_dir) + if info.is_healthy: + return info.pid + return None + + +def hermetic_env( + *, + data_dir: Path, + profile: str, + llm_base_url: str, + llm_model: str = "cassette-model", + llm_api_key: str = "cassette-key", + extra: Mapping[str, str] | None = None, +) -> dict[str, str]: + """Build a daemon environment with every inherited ``LEAPFLOW_*`` removed. + + The developer's shell almost always exports real credentials and a real data + directory. Inheriting either would let a journey read or write the user's + profile, so the whole namespace is dropped before ours is applied. + """ + env = {k: v for k, v in os.environ.items() if not k.startswith("LEAPFLOW_")} + env.update( + { + "LEAPFLOW_DATA_DIR": str(data_dir), + "LEAPFLOW_PROFILE": profile, + "LEAPFLOW_MOCK_HOST": "1", + "LEAPFLOW_LLM_API_KEY": llm_api_key, + "LEAPFLOW_LLM_BASE_URL": llm_base_url, + "LEAPFLOW_LLM_MODEL": llm_model, + "LEAPFLOW_LLM_MAX_RETRIES": "2", + "LEAPFLOW_LLM_CONTEXT_LENGTH": "32768", + "LEAPFLOW_LOG_LEVEL": "INFO", + "LEAPFLOW_DAEMON_LOG_LEVEL": "INFO", + "LEAPFLOW_DAEMON_MAX_CONCURRENT_TURNS": "3", + # The aux/VLM providers share the proxy so no journey can reach the + # network through a secondary client. + "LEAPFLOW_LLM_AUX_BASE_URL": llm_base_url, + "LEAPFLOW_LLM_AUX_API_KEY": llm_api_key, + "LEAPFLOW_VLM_BASE_URL": llm_base_url, + "LEAPFLOW_VLM_API_KEY": llm_api_key, + } + ) + if extra: + env.update({str(k): str(v) for k, v in extra.items()}) + return env + + +class LeapdStartupError(RuntimeError): + """Raised when the daemon never reports a healthy socket.""" + + +class ConcurrentJourneyError(RuntimeError): + """Raised when another run of the same journey already owns its scratch root.""" + + +@dataclass +class Leapd: + """A running ``leapd`` subprocess plus the paths and clients to drive it.""" + + data_dir: Path + profile: str + runtime_dir: Path + env: dict[str, str] + profile_layout: Any = None + process: subprocess.Popen[bytes] | None = None + _workspaces: dict[str, Path] = field(default_factory=dict) + + @property + def sock_path(self) -> Path: + """Unix socket the daemon listens on.""" + return self.runtime_dir / "leapd.sock" + + @property + def log_path(self) -> Path: + """Daemon log file, captured from stdout and stderr.""" + return self.runtime_dir / "leapd.log" + + @property + def audit_log_path(self) -> Path: + """Recovery audit trail, resolved through the profile layout. + + Read from the layout rather than assembled by hand: managed paths are a + product contract, and a journey that hardcodes one stops verifying the + real location the moment the layout moves. + """ + return Path(self.profile_layout.audit_log_path) + + def client(self, *, timeout_s: float = 120.0) -> DaemonClient: + """Return an RPC client for this daemon.""" + return DaemonClient(self.sock_path, timeout_s=timeout_s) + + def info(self) -> DaemonInfo: + """Discover current lifecycle state from the runtime directory.""" + return DaemonInfo.discover(self.runtime_dir) + + def tail_log(self, limit: int = 60) -> str: + """Return the last ``limit`` log lines, or a note when unavailable. + + Cross-process failures are undiagnosable without this: the assertion + happens in the test process while the cause was logged in the daemon's. + """ + try: + lines = self.log_path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return f"(no daemon log at {self.log_path})" + return "\n".join(lines[-limit:]) + + def workspace(self, name: str) -> Path: + """Create (once) and return an isolated workspace directory.""" + existing = self._workspaces.get(name) + if existing is not None: + return existing + path = self.data_dir.parent / "workspaces" / name + path.mkdir(parents=True, exist_ok=True) + self._workspaces[name] = path + return path + + def stop(self) -> None: + """Shut the daemon down gracefully, escalating only if it hangs.""" + if self.process is None: + return + proc = self.process + self.process = None + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=STOP_TIMEOUT_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=STOP_TIMEOUT_S) + cleanup_stale(self.runtime_dir) + + +def start_leapd( + *, + root: Path, + llm_base_url: str, + profile: str = "default", + llm_model: str = "cassette-model", + extra_env: Mapping[str, str] | None = None, + ready_timeout_s: float = READY_TIMEOUT_S, +) -> Leapd: + """Start a real leapd process rooted at ``root`` and wait until it is healthy. + + Args: + root: Scratch directory; the profile tree is created beneath ``root/data``. + llm_base_url: Cassette-proxy base URL every provider is pointed at. + profile: Profile id to create and activate. + llm_model: Model name recorded in cassette fingerprints. + extra_env: Additional ``LEAPFLOW_*`` overrides for this journey. + ready_timeout_s: How long to wait for a healthy socket. + + Returns: + A :class:`Leapd` handle owning the process. + + Raises: + LeapdStartupError: The socket never became healthy; the daemon log tail + is included, since the cause is only ever in the child's log. + """ + data_dir = root / "data" + data_dir.mkdir(parents=True, exist_ok=True) + profile_layout = build_layout(data_dir).ensure(profile_id=profile) + runtime_dir = profile_layout.runtime_dir + runtime_dir.mkdir(parents=True, exist_ok=True) + cleanup_stale(runtime_dir) + + sock_path = runtime_dir / "leapd.sock" + if len(str(sock_path)) > MAX_SOCKET_PATH_LEN: + raise LeapdStartupError( + f"daemon socket path is {len(str(sock_path))} bytes, over the " + f"{MAX_SOCKET_PATH_LEN}-byte Unix socket limit:\n {sock_path}\n" + "Start the journey from a shorter root (see journey_root()); a " + "deep pytest tmp_path cannot host a daemon socket." + ) + + env = hermetic_env( + data_dir=data_dir, + profile=profile, + llm_base_url=llm_base_url, + llm_model=llm_model, + extra=extra_env, + ) + daemon = Leapd( + data_dir=data_dir, + profile=profile, + runtime_dir=runtime_dir, + env=env, + profile_layout=profile_layout, + ) + + log_file = daemon.log_path.open("ab") + try: + process = subprocess.Popen( + [sys.executable, "-m", "leapflow", "--mock-host", "daemon", "serve", "--internal"], + stdin=subprocess.DEVNULL, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env, + cwd=str(root), + start_new_session=True, + close_fds=True, + ) + finally: + log_file.close() + daemon.process = process + + info = wait_ready(runtime_dir, timeout_s=ready_timeout_s) + if not info.is_healthy: + exit_code = process.poll() + daemon.stop() + raise LeapdStartupError( + f"leapd never became healthy within {ready_timeout_s:.0f}s " + f"(exit={exit_code}, socket={daemon.sock_path})\n" + f"--- leapd log tail ---\n{daemon.tail_log()}" + ) + return daemon + + +async def await_for( + predicate: Any, + *, + timeout_s: float = 10.0, + interval_s: float = 0.05, + what: str = "condition", +) -> Any: + """Poll ``predicate`` until it returns a truthy value or the timeout expires. + + Cross-process state (a DuckDB write, a released lease, an exited process) + becomes visible slightly after the RPC returns, so journeys wait on the + observable fact rather than sleeping on a guess. The failure names what was + being waited for and the last value seen. + """ + deadline = time.time() + timeout_s + last: Any = None + while time.time() < deadline: + last = await predicate() + if last: + return last + await asyncio.sleep(max(0.01, interval_s)) + raise AssertionError(f"timed out after {timeout_s:.1f}s waiting for {what} (last={last!r})") diff --git a/tests/conftest.py b/tests/conftest.py index e93e829..956f86f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,41 @@ from leapflow.storage.trajectory_store import TrajectoryStore +# ════════════════════════════════════════════════════════════════ +# Layer markers — applied by path so existing files need no edit +# ════════════════════════════════════════════════════════════════ + +_TESTS_ROOT = Path(__file__).resolve().parent +_EXPLICIT_LAYERS = frozenset({"unit", "component", "e2e", "live"}) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: List[pytest.Item] +) -> None: + """Assign a layer marker to every test based on its location. + + Labelling by path keeps the 1400-case mock suite untouched while still making + the layers selectable: ``tests/journeys/`` is the real end-to-end layer, + ``tests/regression/`` is the always-on incident ledger, and everything else + defaults to ``unit`` unless the file opts into ``component`` itself. + """ + for item in items: + try: + relative = Path(str(item.fspath)).resolve().relative_to(_TESTS_ROOT) + except ValueError: + continue + top = relative.parts[0] if relative.parts else "" + if top == "journeys": + item.add_marker(pytest.mark.e2e) + item.add_marker(pytest.mark.slow) + continue + if top == "regression": + item.add_marker(pytest.mark.invariant) + continue + if not _EXPLICIT_LAYERS.intersection(marker.name for marker in item.iter_markers()): + item.add_marker(pytest.mark.unit) + + # ═══════════════════════════════════════════════════════════════════ # Stub LLM — scripted responses for deterministic integration tests # ═══════════════════════════════════════════════════════════════════ diff --git a/tests/journeys/__init__.py b/tests/journeys/__init__.py new file mode 100644 index 0000000..c0e1949 --- /dev/null +++ b/tests/journeys/__init__.py @@ -0,0 +1 @@ +"""Real end-to-end journeys: coarse, cross-module, always run.""" diff --git a/tests/journeys/conftest.py b/tests/journeys/conftest.py new file mode 100644 index 0000000..3a0e572 --- /dev/null +++ b/tests/journeys/conftest.py @@ -0,0 +1,58 @@ +"""Fixtures for the real end-to-end journey layer. + +Every journey runs against a real ``leapd`` subprocess with the LLM boundary +served by a local cassette proxy. Mode selection is environment-driven so the +same journey bodies serve all four lanes: + +- ``replay`` (default, and what CI PR/main lanes use) — offline, deterministic +- ``seed`` — author committed cassettes from a journey's declared script +- ``record`` — capture real provider traffic into cassettes +- ``live`` — run against a real provider, persisting nothing +""" + +from __future__ import annotations + +from typing import Iterator + +import pytest + +from tests._harness.cassette_proxy import LIVE, RECORD, resolve_mode, upstream_from_env +from tests._harness.journey import JourneyFactory + + +@pytest.fixture(scope="session") +def journey_mode() -> str: + """Resolve and validate the cassette mode once per session.""" + mode = resolve_mode() + if mode in (RECORD, LIVE): + base_url, api_key, model = upstream_from_env() + missing = [ + name + for name, value in ( + ("LEAPFLOW_LLM_BASE_URL", base_url), + ("LEAPFLOW_LLM_API_KEY", api_key), + ("LEAPFLOW_LLM_MODEL", model), + ) + if not value + ] + if missing: + pytest.skip( + f"mode {mode!r} needs a real provider; missing {missing} " + "(the live lane injects them from secrets)" + ) + return mode + + +@pytest.fixture +def journeys(journey_mode: str) -> Iterator[JourneyFactory]: + """Factory that starts journeys and tears them down after the test. + + Note it does not take ``tmp_path``: the factory allocates a deliberately + short scratch root, because a daemon Unix socket cannot live under pytest's + deep temp directory on macOS. + """ + factory = JourneyFactory(journey_mode) + try: + yield factory + finally: + factory.close() diff --git a/tests/journeys/test_r1_conversation.py b/tests/journeys/test_r1_conversation.py new file mode 100644 index 0000000..a6b6b29 --- /dev/null +++ b/tests/journeys/test_r1_conversation.py @@ -0,0 +1,179 @@ +"""R1 — the conversation main line, end to end through a real daemon. + +Phases: first turn → streamed chunks → native tool call → tool result fed back → +final answer → history and usage persisted → second turn continues the session. + +What only this layer can observe: + +- turns land on the *per-session* engine, not on the base template, so + ``status(session_id)`` reports real context usage instead of zero; +- the tool result actually reaches the next model call, across process boundaries; +- history and token accounting survive in DuckDB, written by the daemon process + and read back over RPC. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests._harness.cassette_proxy import answer, scripted, tool_call +from tests._harness.journey import JourneyFactory +from tests._harness.leapd import await_for + +# ── Journey metadata, read by tools/impact.py (AST-parsed, never imported) ── +# +# SUBJECT_PATHS names the source areas this journey actually exercises, so a +# label-triggered live run can pick the journeys a change could plausibly break +# instead of paying for all of them. Declared here rather than in a central table +# because it belongs next to the assertions it describes and moves with them. +SUBJECT_PATHS = ( + "src/leapflow/engine/", + "src/leapflow/llm/", + "src/leapflow/tools/", + "src/leapflow/daemon/", + "src/leapflow/memory/", + "src/leapflow/storage/", +) + +# Running this against a real provider adds signal: it is the only journey that +# exercises real tool-calling and real streaming end to end. +LIVE_SIGNAL = True + +WORKSPACE_FILE = "invoice.txt" +WORKSPACE_CONTENT = "Invoice 42\nTotal: 128.50 USD\n" + + +async def _drive(client: Any, message: str, *, session_id: str, workspace: str) -> list[Any]: + """Run one turn and return every stream event it produced.""" + events: list[Any] = [] + async for event in client.engine_chat( + message, session_id=session_id, workspace_root=workspace + ): + events.append(event) + return events + + +def _text_of(events: list[Any], *types: str) -> str: + """Concatenate the content of events of the given types.""" + return "".join(event.content for event in events if event.type in types) + + +@pytest.mark.asyncio +async def test_r1_conversation_main_line(journeys: JourneyFactory) -> None: + """A full conversation: stream, call a tool, answer, persist, continue.""" + journey = journeys( + "r1_conversation", + script=scripted( + answer("Hello from LeapFlow."), + tool_call("file_read", path=WORKSPACE_FILE), + answer("The invoice total is 128.50 USD."), + answer("Yes, that is the same invoice."), + ), + deadline_s=90.0, + # Four turns. Observed against a real provider: 4 calls when the model + # went straight to file_read, 7 when it globbed first. The ceiling leaves + # room for that swing without leaving room for a runaway loop. + max_llm_calls=12, + # Observed 36k-65k tokens across real runs; ~9k prompt tokens per call is + # the floor set by the system prompt plus tool schemas. A jump past this + # means prompt assembly grew, which is exactly what would otherwise raise + # the live lane's bill in silence. + max_llm_tokens=140_000, + ) + workspace = journey.workspace("main") + (workspace / WORKSPACE_FILE).write_text(WORKSPACE_CONTENT, encoding="utf-8") + session_id = "r1-session" + client = journey.client() + + with journey.phase("boot: daemon reports itself without inventing a session"): + status = await client.status() + assert status["pid"] > 0 + assert status["profile"] == "default" + assert status["session_id"] == "", ( + "a caller that named no session must not be handed somebody else's identity" + ) + + with journey.phase("first turn: streamed answer"): + events = await _drive( + client, "Say hello.", session_id=session_id, workspace=str(workspace) + ) + assert _text_of(events, "chunk", "final"), f"no answer text in {[e.type for e in events]}" + assert not [e for e in events if e.type == "error"], ( + f"turn reported errors: {[e.content for e in events if e.type == 'error']}" + ) + + with journey.phase("session state: reported from the session engine, not the template"): + scoped = await client.status(session_id) + assert scoped["session_id"] == session_id + assert scoped["context_used"] > 0, ( + "context usage read as zero — the reporting path resolved the base " + "engine template instead of the session engine" + ) + assert scoped["llm_context_length"] > 0 + + with journey.phase("tool turn: native call executes and its result feeds back"): + events = await _drive( + client, + f"Use the file_read tool on {WORKSPACE_FILE} and report the total.", + session_id=session_id, + workspace=str(workspace), + ) + assert not [e for e in events if e.type == "error"], ( + f"tool turn failed: {[e.content for e in events if e.type == 'error']}" + ) + started = [e.content for e in events if e.type == "tool_start"] + completed = [e.content for e in events if e.type == "tool_complete"] + + if journey.is_live: + # Which tool a real model reaches for is its own decision, so the + # live lane asserts only that tool dispatch works when it happens. + # The strict form below runs on every push, in replay. + assert set(started) == set(completed), ( + f"a tool started but never completed: started={started} " + f"completed={completed}" + ) + else: + assert "file_read" in started, f"tool was never started: {[e.type for e in events]}" + assert "file_read" in completed, f"tool never completed: {started}" + fed_back = journey.proxy.stats.prompts_containing("128.50") + assert fed_back, ( + "the file content never reached a subsequent model call — the tool " + "result did not make it back into the conversation" + ) + + with journey.phase("persistence: history and usage survive in the daemon's store"): + history = await await_for( + lambda: _history_of(client, session_id), + timeout_s=10.0, + what="persisted history", + ) + roles = [str(message.get("role", "")) for message in history] + assert roles.count("user") >= 2, f"user turns missing from history: {roles}" + assert "assistant" in roles, f"assistant turns missing from history: {roles}" + + usage = await client.usage_summary() + assert usage, "usage summary is empty" + + with journey.phase("continuation: the same session keeps its context"): + events = await _drive( + client, + "Is that the same invoice?", + session_id=session_id, + workspace=str(workspace), + ) + assert not [e for e in events if e.type == "error"] + final_status = await client.status(session_id) + assert final_status["session_id"] == session_id + assert final_status["context_used"] >= scoped["context_used"], ( + "context usage went backwards across turns of one session" + ) + + journey.finish() + + +async def _history_of(client: Any, session_id: str) -> list[dict[str, Any]]: + """Fetch persisted messages, returning [] until the write is visible.""" + payload = await client.session_history(session_id=session_id) + return list(payload.get("messages") or []) diff --git a/tests/journeys/test_r2_isolation.py b/tests/journeys/test_r2_isolation.py new file mode 100644 index 0000000..2917e7b --- /dev/null +++ b/tests/journeys/test_r2_isolation.py @@ -0,0 +1,160 @@ +"""R2 — concurrency and session identity across two workspaces on one daemon. + +Several TUIs in different workspaces sharing one leapd is a supported way to use +LeapFlow, not an edge case, and it is the scenario a single-session test cannot +observe at all. Every incident in this area shipped with a green suite: a second +client adopted the first's session id, sent it with its own workspace, and was +rejected on every turn. + +Phases: two sessions in two workspaces → interleaved turns → cross-read status, +history and usage → an anonymous status reveals nobody → cross-workspace reuse is +refused with an actionable message. +""" + +from __future__ import annotations + +import asyncio +import os +from typing import Any + +import pytest + +from tests._harness.cassette_proxy import answer, scripted +from tests._harness.journey import JourneyFactory + +# Journey metadata read by tools/impact.py (see test_r1_conversation.py). +SUBJECT_PATHS = ( + "src/leapflow/daemon/", + "src/leapflow/engine/", + "src/leapflow/cli/", +) + +# Concurrency and identity behaviour can shift with real provider latency, which +# is exactly the condition under which cross-client leakage appeared. +LIVE_SIGNAL = True + +SESSION_A = "r2-client-a" +SESSION_B = "r2-client-b" + + +async def _turn(client: Any, message: str, *, session_id: str, workspace: str) -> list[Any]: + """Run one turn to completion and return its stream events.""" + events: list[Any] = [] + async for event in client.engine_chat( + message, session_id=session_id, workspace_root=workspace + ): + events.append(event) + return events + + +@pytest.mark.asyncio +async def test_r2_two_workspaces_stay_isolated(journeys: JourneyFactory) -> None: + """Two clients on one daemon never see each other's session, usage, or turns.""" + journey = journeys( + "r2_isolation", + script=scripted( + answer("Workspace A acknowledged."), + answer("Workspace B acknowledged."), + answer("Still workspace A."), + answer("Still workspace B."), + ), + deadline_s=90.0, + # Six turns across two clients. Observed 8 calls against a real provider. + max_llm_calls=14, + # Observed 72k tokens against a real provider (~8.9k prompt per call). + max_llm_tokens=160_000, + ) + workspace_a = journey.workspace("alpha") + workspace_b = journey.workspace("beta") + client_a = journey.client() + client_b = journey.client() + + with journey.phase("cross-process: the daemon really is another process"): + boot = await client_a.status() + assert boot["pid"] != os.getpid(), ( + "status() reported this test's pid — the journey is not exercising a " + "separate daemon process, so no cross-process contract is being tested" + ) + assert boot["session_id"] == "", ( + "an anonymous caller must receive no session identity; reporting one " + "is how a fresh client adopts another client's session" + ) + + with journey.phase("first turns: each client opens its own session"): + events_a = await _turn( + client_a, "Hello from A.", session_id=SESSION_A, workspace=str(workspace_a) + ) + events_b = await _turn( + client_b, "Hello from B.", session_id=SESSION_B, workspace=str(workspace_b) + ) + for label, events in (("A", events_a), ("B", events_b)): + errors = [event.content for event in events if event.type == "error"] + assert not errors, f"client {label} turn failed: {errors}" + + with journey.phase("identity: each status reports only its own caller"): + status_a = await client_a.status(SESSION_A) + status_b = await client_b.status(SESSION_B) + assert status_a["session_id"] == SESSION_A + assert status_b["session_id"] == SESSION_B + assert status_a["session_id"] != status_b["session_id"] + + with journey.phase("anonymous status still reveals nobody after both are live"): + anonymous = await client_a.status() + assert anonymous["session_id"] == "", ( + "with two live sessions the daemon answered an unscoped status with a " + "session identity — that value belongs to whichever session ran last" + ) + assert anonymous.get("context_used", 0) == 0, ( + "unscoped status reported per-session context usage that belongs to " + "some other client" + ) + + with journey.phase("history: neither client can read the other's conversation"): + history_a = await client_a.session_history(session_id=SESSION_A) + history_b = await client_b.session_history(session_id=SESSION_B) + blob_a = str(history_a.get("messages") or []) + blob_b = str(history_b.get("messages") or []) + assert "Hello from A." in blob_a + assert "Hello from B." in blob_b + assert "Hello from B." not in blob_a, "client A can read client B's conversation" + assert "Hello from A." not in blob_b, "client B can read client A's conversation" + + with journey.phase("concurrent turns: interleaving does not cross-contaminate"): + results = await asyncio.gather( + _turn(client_a, "Second A turn.", session_id=SESSION_A, workspace=str(workspace_a)), + _turn(client_b, "Second B turn.", session_id=SESSION_B, workspace=str(workspace_b)), + ) + for label, events in zip(("A", "B"), results): + errors = [event.content for event in events if event.type == "error"] + assert not errors, f"concurrent turn for client {label} failed: {errors}" + + after_a = await client_a.status(SESSION_A) + after_b = await client_b.status(SESSION_B) + assert after_a["session_id"] == SESSION_A + assert after_b["session_id"] == SESSION_B + + final_a = await client_a.session_history(session_id=SESSION_A) + assert "Second B turn." not in str(final_a.get("messages") or []), ( + "a concurrent turn from client B landed in client A's session" + ) + + with journey.phase("workspace binding: reuse from another workspace is refused"): + events = await _turn( + client_b, "Steal A's session.", session_id=SESSION_A, workspace=str(workspace_b) + ) + errors = [event for event in events if event.type == "error"] + assert errors, ( + "reusing session A from workspace B was accepted — a session is bound " + "to the workspace of its first request" + ) + metadata = errors[0].metadata or {} + assert metadata.get("workspace_mismatch") is True, ( + f"refusal was not classified as a workspace mismatch: {metadata}" + ) + assert metadata.get("session_id") == SESSION_A + assert str(workspace_a) in str(metadata.get("expected_workspace_root", "")), ( + "the refusal must name the workspace the session is bound to, since " + "an explicit --resume into another workspace is the only legitimate cause" + ) + + journey.finish() diff --git a/tests/journeys/test_r3_control_plane.py b/tests/journeys/test_r3_control_plane.py new file mode 100644 index 0000000..688aafe --- /dev/null +++ b/tests/journeys/test_r3_control_plane.py @@ -0,0 +1,205 @@ +"""R3 — the control plane: slash commands, layered config, secrets, cancellation. + +`leap config` / `/config` is the only sanctioned way to change durable settings, +so its whole path has to hold across a process boundary: the mutation must be +written to the profile, reloaded by the running daemon, and echoed back on the +mutation payload — a daemon-mode client is a separate process and cannot observe +a reload it was not told about. Secrets must land in the vault as refs, never as +plaintext on disk. + +This journey needs no LLM semantics, so it is replay-only: the live lane would +add cost without adding signal. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests._harness.cassette_proxy import answer, scripted +from tests._harness.journey import JourneyFactory +from tests._harness.leapd import await_for + +# Journey metadata read by tools/impact.py (see test_r1_conversation.py). +SUBJECT_PATHS = ( + "src/leapflow/config.py", + "src/leapflow/config_loader.py", + "src/leapflow/config_service.py", + "src/leapflow/layout.py", + "src/leapflow/security/", + "src/leapflow/cli/commands/", +) + +# No LLM semantics: every assertion is about config layering, the vault, and +# slash-command payloads. A live run would spend tokens for no extra signal. +LIVE_SIGNAL = False + +SESSION = "r3-control-plane" +SECRET_VALUE = "sk-r3-journey-secret-value" + +# A durable, hot-reloadable integer the journey harness does not pin through the +# environment. Env is the highest-priority config layer, so a key the harness +# exports could never change effective value here — which would silently turn +# this phase into a no-op assertion. +MUTABLE_KEY = "memory.working_max_tokens" +MUTATED_VALUE = "4096" + +# Pinned by the harness on purpose (cassette fingerprints depend on it), and used +# below to assert the layering contract rather than to test mutation. +ENV_PINNED_KEY = "llm.context_length" + + +def _plaintext_hits(root: Path, needle: str) -> list[str]: + """Return every readable file under ``root`` containing ``needle``.""" + hits: list[str] = [] + for path in root.rglob("*"): + if not path.is_file(): + continue + try: + content = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + if needle in content: + hits.append(str(path.relative_to(root))) + return hits + + +@pytest.mark.asyncio +async def test_r3_control_plane(journeys: JourneyFactory) -> None: + """Slash commands observe, mutate, reload, and cancel through the daemon.""" + journey = journeys( + "r3_control_plane", + script=scripted(answer("Acknowledged.")), + deadline_s=90.0, + # One closing turn; the rest of the journey is pure control plane. + max_llm_calls=6, + max_llm_tokens=80_000, + ) + workspace = journey.workspace("ctrl") + client = journey.client() + + with journey.phase("observe: /status and /usage answer without a turn"): + status = await client.command_execute("status", session_id=SESSION) + assert status.get("ok") is not False, f"/status failed: {status}" + usage = await client.command_execute("usage", session_id=SESSION) + assert usage.get("ok") is not False, f"/usage failed: {usage}" + + with journey.phase("discover: the config catalog is self-describing"): + keys = await client.command_execute("config", "keys", session_id=SESSION) + assert keys.get("ok") is True, f"/config keys failed: {keys}" + assert keys.get("mode") == "keys" + assert keys.get("sources"), "the writable-key catalog is empty" + + listing = await client.command_execute("config", "list llm", session_id=SESSION) + assert listing.get("mode") == "list" + fields = listing.get("fields") or [] + assert fields, "/config list llm returned no fields" + described = {str(field.get("key", "")) for field in fields} + assert "llm.model" in described, f"llm.model is not discoverable: {sorted(described)}" + + with journey.phase("detail: /config show names one field precisely"): + detail = await client.command_execute("config", "show llm.model", session_id=SESSION) + assert detail.get("mode") == "show_detail" + field = detail.get("field") or {} + assert field.get("key") == "llm.model" + + with journey.phase("mutate: /config set persists, reloads, and echoes runtime state"): + before = await _get_value(client, MUTABLE_KEY) + assert before != MUTATED_VALUE, "fixture would not change anything" + + mutation = await client.command_execute( + "config", f"set {MUTABLE_KEY} {MUTATED_VALUE}", session_id=SESSION + ) + assert mutation.get("ok") is True, f"/config set failed: {mutation}" + assert mutation.get("mode") == "mutation" + assert MUTABLE_KEY in (mutation.get("changed_keys") or []), ( + f"the mutation reported no changed key: {mutation}" + ) + + # A daemon-mode client is a separate process: it can only learn runtime + # state from what the reply carries, so the mutation payload must ship the + # values the status bar renders. + live = await client.status(SESSION) + assert mutation.get("model") == live.get("model"), ( + f"mutation payload model {mutation.get('model')!r} disagrees with the " + f"daemon's {live.get('model')!r}" + ) + assert mutation.get("llm_context_length") == live.get("llm_context_length"), ( + "the mutation payload did not carry the runtime context length the " + "status bar renders" + ) + + with journey.phase("reload: the running daemon serves the new value"): + served = await await_for( + lambda: _value_equals(client, MUTABLE_KEY, MUTATED_VALUE), + timeout_s=15.0, + what=f"daemon to serve the reloaded {MUTABLE_KEY}", + ) + assert served, ( + f"{MUTABLE_KEY} was written but the running daemon never reloaded it " + "(if the harness now exports it as LEAPFLOW_*, the process override " + "outranks the durable write and this key can no longer be tested here)" + ) + + with journey.phase("layering: a process override outranks a durable write"): + # The documented precedence is env > workspace > profile > user, and it is a + # real footgun: a user who exported LEAPFLOW_* once sees `/config set` + # report success while nothing changes. The write must still be accepted, + # and the effective value must still come from the override. + pinned_before = await _get_value(client, ENV_PINNED_KEY) + result = await client.command_execute( + "config", f"set {ENV_PINNED_KEY} 24576", session_id=SESSION + ) + assert result.get("ok") is True, f"durable write refused: {result}" + pinned_after = await _get_value(client, ENV_PINNED_KEY) + assert pinned_after == pinned_before != "24576", ( + f"{ENV_PINNED_KEY} is pinned to {pinned_before!r} by the harness " + f"environment but a durable write changed the effective value to " + f"{pinned_after!r} — config precedence changed" + ) + + with journey.phase("secrets: credentials become refs, never plaintext on disk"): + secret = await client.command_execute( + "config", f"secret set llm.primary.api_key {SECRET_VALUE}", session_id=SESSION + ) + assert secret.get("ok") is True, f"/config secret set failed: {secret}" + + leaks = _plaintext_hits(journey.daemon.data_dir, SECRET_VALUE) + assert leaks == [], ( + f"the secret was written as plaintext to {leaks} — long-lived " + "credentials must be vault-encrypted and referenced as secret:// refs" + ) + + with journey.phase("cancel: an idle cancel is safe and reports honestly"): + cancelled = await client.engine_cancel() + assert isinstance(cancelled, bool), f"engine.cancel returned {cancelled!r}" + + with journey.phase("still usable: a turn works after the control-plane churn"): + events: list[Any] = [] + async for event in client.engine_chat( + "Anything to report?", session_id=SESSION, workspace_root=str(workspace) + ): + events.append(event) + errors = [event.content for event in events if event.type == "error"] + assert not errors, f"turn broke after config mutations: {errors}" + + journey.finish() + + +async def _get_value(client: Any, key: str) -> str: + """Return the effective ``/config get`` value for ``key`` as reported. + + ``/config get`` renders values as display strings, so comparisons here stay + textual rather than guessing at the underlying Python type. + """ + payload = await client.command_execute("config", f"get {key}", session_id=SESSION) + values = payload.get("values") or [] + assert values, f"/config get {key} returned nothing: {payload}" + return str(values[0].get("value")) + + +async def _value_equals(client: Any, key: str, expected: Any) -> bool: + """True once the daemon serves ``expected`` for ``key``.""" + return await _get_value(client, key) == str(expected) diff --git a/tests/journeys/test_r4_recovery.py b/tests/journeys/test_r4_recovery.py new file mode 100644 index 0000000..3500e31 --- /dev/null +++ b/tests/journeys/test_r4_recovery.py @@ -0,0 +1,206 @@ +"""R4 — failure and recovery, driven by real provider wire semantics. + +Every failure here arrives as an actual HTTP response through the real ``openai`` +client, so the classifier reads the same bytes a provider would send. That is the +part a mock cannot check: a hand-raised exception proves the handler runs, not +that the *real* error is recognised as the category it belongs to. + +Phases: rate limit is retried → server error is retried → context overflow is +transformed and retried → an unrecoverable failure halts with something the user +can act on, and leaves an audit trail. + +Deliberately *not* here (they belong to other layers per the duty matrix): +exception-type classification of local defects, per-category budget arithmetic, +and side-effect gating decisions are table-and-branch concerns for the mock +layer; truncated-SSE handling is a provider-parsing concern covered where the +streaming path actually runs (``tests/test_journey_harness.py``), since the +engine's native-tool round is non-streaming. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from tests._harness.cassette import ( + context_overflow_response, + error_response, + rate_limited_response, + server_error_response, +) +from tests._harness.cassette_proxy import answer, scripted +from tests._harness.journey import Journey, JourneyFactory +from tests._harness.leapd import await_for + +# Journey metadata read by tools/impact.py (see test_r1_conversation.py). +SUBJECT_PATHS = ( + "src/leapflow/engine/", + "src/leapflow/llm/", +) + +# Cannot run live at all: every response it asserts on is an injected failure, +# and a forwarding mode sends each request upstream instead. The factory also +# enforces this at runtime via requires_scripted_responses. +LIVE_SIGNAL = False + +SESSION = "r4-recovery" + + +async def _turn(journey: Journey, client: Any, message: str, workspace: str) -> list[Any]: + """Run one turn and return its stream events.""" + events: list[Any] = [] + async for event in client.engine_chat( + message, session_id=SESSION, workspace_root=workspace + ): + events.append(event) + return events + + +def _errors(events: list[Any]) -> list[Any]: + return [event for event in events if event.type == "error"] + + +def _answer_text(events: list[Any]) -> str: + return "".join(event.content for event in events if event.type in ("chunk", "final")) + + +def _audit_entries(path: Path) -> list[dict[str, Any]]: + """Read the recovery audit trail the daemon wrote to ``path``.""" + if not path.is_file(): + return [] + entries: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError: + continue + return entries + + +@pytest.mark.asyncio +async def test_r4_failure_and_recovery(journeys: JourneyFactory) -> None: + """Provider failures are classified, retried where safe, and halt actionably.""" + journey = journeys( + "r4_recovery", + script=scripted( + # Phase: transient rate limit, then the real answer. + rate_limited_response(), + answer("Recovered after a rate limit."), + # Phase: transient server error, then the real answer. + server_error_response(), + answer("Recovered after a server error."), + # Phase: context overflow, then the real answer once transformed. + context_overflow_response(), + answer("Recovered after compressing context."), + # Phase: a permanent, non-retryable failure for every attempt. + error_response( + 400, + code="unsupported_value", + message="The requested configuration is not supported by this model", + ), + ), + # Every response here is an injected failure, so this journey is only + # meaningful against recordings: a forwarding mode sends each request + # upstream and the failures never happen. + requires_scripted_responses=True, + deadline_s=120.0, + # Eight scripted responses plus provider-level retries. A higher count + # means recovery stopped converging. + max_llm_calls=20, + # Injected failures carry no usage, so the real spend here is the retried + # successes; replay-only, so this ceiling guards loop growth, not cost. + max_llm_tokens=100_000, + ) + workspace = str(journey.workspace("recover")) + client = journey.client() + + with journey.phase("rate limit: retried transparently, user still gets an answer"): + before = journey.proxy.stats.call_count + events = await _turn(journey, client, "Summarize the situation.", workspace) + assert not _errors(events), f"a retryable 429 surfaced as an error: {_errors(events)}" + assert _answer_text(events), "no answer text after recovering from a 429" + assert journey.proxy.stats.call_count > before + 1, ( + "the 429 was never retried — only one provider call was made" + ) + + with journey.phase("server error: retried transparently"): + before = journey.proxy.stats.call_count + events = await _turn(journey, client, "And now?", workspace) + assert not _errors(events), f"a retryable 500 surfaced as an error: {_errors(events)}" + assert journey.proxy.stats.call_count > before + 1, "the 500 was never retried" + + with journey.phase("context overflow: transformed and retried, not abandoned"): + before = journey.proxy.stats.call_count + events = await _turn(journey, client, "Keep going with more context.", workspace) + assert not _errors(events), ( + f"a context overflow was treated as terminal: {[e.content for e in _errors(events)]}" + ) + assert _answer_text(events), "no answer after context-overflow recovery" + assert journey.proxy.stats.call_count > before + 1, ( + "the overflow produced no follow-up call — nothing was transformed or retried" + ) + + with journey.phase("unrecoverable: halts with something the user can act on"): + events = await _turn(journey, client, "Do the impossible thing.", workspace) + errors = _errors(events) + assert errors, "a permanent provider failure produced no error event" + + terminal = errors[-1] + assert terminal.content.strip(), "the terminal error carried no message at all" + + interaction = (terminal.metadata or {}).get("interaction") or {} + if interaction: + assert interaction.get("title"), f"interaction without a title: {interaction}" + assert interaction.get("suggested_actions"), ( + "a halt must name the next step, not just the failure: " + f"{interaction}" + ) + assert interaction.get("resumption_key"), ( + "a resumable halt needs a resumption key so the client can continue" + ) + else: + # No InteractionRequest was attached; the message itself must then be + # more than internal jargon, since it is all the user gets. + assert "No applicable recovery strategy" not in terminal.content, ( + "the user was shown a raw internal reason with no guidance: " + f"{terminal.content!r}" + ) + + with journey.phase("evidence: recovery decisions are recorded for after the fact"): + audit_path = journey.daemon.audit_log_path + entries = await await_for( + lambda: _await_audit(audit_path), + timeout_s=10.0, + what=f"recovery audit entries in {audit_path}", + ) + classified = [ + entry for entry in entries if str(entry.get("failure_category") or "").strip() + ] + assert classified, ( + f"audit entries carry no failure classification: {entries[:3]}" + ) + assert any(str(entry.get("strategy_key") or "").strip() for entry in classified), ( + "no audit entry names the strategy that decided the outcome, so an " + f"incident could not be reconstructed: {classified[:3]}" + ) + assert any(str(entry.get("reason") or "").strip() for entry in classified), ( + "audit entries carry no explainable reason" + ) + + with journey.phase("still usable: the session survives the failure sequence"): + status = await client.status(SESSION) + assert status["session_id"] == SESSION + assert status["context_used"] > 0 + + journey.finish() + + +async def _await_audit(path: Path) -> list[dict[str, Any]]: + """Return audit entries once the daemon has flushed at least one.""" + return _audit_entries(path) diff --git a/tests/journeys/test_r5_learning.py b/tests/journeys/test_r5_learning.py new file mode 100644 index 0000000..7a5863c --- /dev/null +++ b/tests/journeys/test_r5_learning.py @@ -0,0 +1,150 @@ +"""R5 — the learning loop: teach → record → stop → distill → skill visible. + +Progressive Trust starts at recording, so the loop only means something if each +stage survives a process boundary: the teaching session is opened inside the +daemon, steps accumulate on the daemon-owned session, distillation runs there, +and the resulting skill must be readable back through a separate RPC call. An +in-process test can hold all of that in one object graph and prove none of it. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from tests._harness.cassette_proxy import answer, scripted +from tests._harness.journey import Journey, JourneyFactory +from tests._harness.leapd import await_for + +# Journey metadata read by tools/impact.py (see test_r1_conversation.py). +SUBJECT_PATHS = ( + "src/leapflow/recording/", + "src/leapflow/learning/", + "src/leapflow/analysis/", + "src/leapflow/skills/", + "src/leapflow/engine/session.py", + "src/leapflow/storage/", +) + +# Distillation quality depends on the real model's output, so running it live is +# the only way to see whether a real answer still distils into a usable skill. +LIVE_SIGNAL = True + +SESSION = "r5-learning" + + +async def _cmd(client: Any, name: str, args: str = "") -> dict[str, Any]: + """Execute one slash command for this journey's session.""" + return await client.command_execute(name, args, session_id=SESSION) + + +async def _turn(journey: Journey, client: Any, message: str, workspace: str) -> list[Any]: + """Run one turn so the teaching session has something to record.""" + events: list[Any] = [] + async for event in client.engine_chat( + message, session_id=SESSION, workspace_root=workspace + ): + events.append(event) + return events + + +@pytest.mark.asyncio +async def test_r5_learning_loop(journeys: JourneyFactory) -> None: + """Teaching records, stops, distills, and leaves an inspectable skill library.""" + journey = journeys( + "r5_learning", + script=scripted( + answer("Noted the first step."), + answer("Noted the second step."), + answer( + '{"title": "Tidy invoices", ' + '"trigger_phrases": ["tidy invoices", "sort invoices"], ' + '"steps": ["List the invoice folder", "Classify by month", "Move into folders"], ' + '"parameters": [{"name": "path", "description": "invoice folder"}], ' + '"pre_conditions": [], "confidence": 0.7}' + ), + # The final entry repeats for every later call, so it must be a benign + # answer. Leaving the distillation JSON last made the closing turn + # re-read it as a reply and loop until its iteration budget ran out. + answer("Done — nothing further needed."), + ), + deadline_s=120.0, + # Three turns plus a possible background distillation call. Observed 3 + # calls against a real provider. A jump here is the signature of the loop + # that used to re-read the distillation payload as a reply and run to its + # iteration cap. + max_llm_calls=12, + # Observed 27k tokens against a real provider. + max_llm_tokens=140_000, + ) + workspace = str(journey.workspace("learn")) + client = journey.client() + + with journey.phase("baseline: the skill library is inspectable and starts empty"): + listing = await _cmd(client, "skill", "list") + assert listing.get("ok") is True, f"/skill list failed: {listing}" + assert listing.get("view") == "skill_list" + baseline = {str(item.get("name")) for item in listing.get("skills") or []} + + with journey.phase("open: a turn must exist before teaching can attach to it"): + events = await _turn(journey, client, "Let me show you something.", workspace) + assert not [event for event in events if event.type == "error"], ( + "the seeding turn failed, so there is no session to teach against" + ) + + with journey.phase("record: /teach start enters learning mode"): + started = await _cmd(client, "teach start", "tidy the invoice folder") + assert started.get("session_mode") == "learning", ( + f"/teach start did not enter learning mode: {started}" + ) + + with journey.phase("status: the daemon reports the recording session honestly"): + status = await _cmd(client, "teach status") + assert status.get("ok") is not False, f"/teach status failed: {status}" + + with journey.phase("annotate: the user's own words are captured"): + annotated = await _cmd(client, "annotate", "invoices go into per-month folders") + assert annotated.get("ok") is not False, f"/annotate failed: {annotated}" + + with journey.phase("steps: activity accumulates on the daemon-owned session"): + await _turn(journey, client, "Now sort them by month.", workspace) + + with journey.phase("stop: recording ends and reports what it captured"): + stopped = await _cmd(client, "teach stop") + assert stopped.get("ok") is True, f"/teach stop failed: {stopped}" + assert stopped.get("session_mode") != "learning", ( + f"still in learning mode after /teach stop: {stopped}" + ) + assert stopped.get("message"), "/teach stop said nothing about what it recorded" + + with journey.phase("persist: the trajectory survives in the daemon's store"): + trajectories = await await_for( + lambda: _trajectory_files(journey), + timeout_s=20.0, + what="a persisted teaching trajectory", + ) + assert trajectories, "teaching produced no durable trajectory" + + with journey.phase("library: skills remain listable, and nothing was corrupted"): + after = await _cmd(client, "skill", "list") + assert after.get("ok") is True, f"/skill list failed after teaching: {after}" + names = {str(item.get("name")) for item in after.get("skills") or []} + assert names >= baseline, ( + f"skills disappeared during the learning loop: {baseline - names}" + ) + for entry in after.get("skills") or []: + assert 0.0 <= float(entry.get("confidence", 0.0)) <= 1.0, ( + f"skill confidence is out of range: {entry}" + ) + + with journey.phase("still usable: a normal turn works after teaching"): + events = await _turn(journey, client, "Thanks, that is all.", workspace) + assert not [event for event in events if event.type == "error"] + + journey.finish() + + +async def _trajectory_files(journey: Journey) -> list[str]: + """Return DuckDB stores the daemon created, once any exist on disk.""" + return [str(path.name) for path in journey.daemon.data_dir.rglob("*.duckdb")] diff --git a/tests/journeys/test_r6_lifecycle.py b/tests/journeys/test_r6_lifecycle.py new file mode 100644 index 0000000..cfeae7b --- /dev/null +++ b/tests/journeys/test_r6_lifecycle.py @@ -0,0 +1,161 @@ +"""R6 — daemon runtime lifecycle: start, report, restart, stop, recover from stale state. + +Lifecycle is only meaningful across processes: a PID file, a Unix socket and a +metadata file are all real artefacts on disk, and "restart" means the old process +is gone and a new one owns them. A stale socket left by a crashed daemon must not +block the next start, and version reporting must describe the process actually +answering — not the client asking. + +Scope note: inbound gateway signal classification is *not* here. The gateway RPCs +are not implemented in this daemon phase, so a journey could only assert the +NotImplementedError; normalization, SNR filtering, trigger policy and +self-message filtering are already covered where they run, in the mock layer +(``test_feishu_event_normalizer.py``, ``test_gateway_consumer_loop.py``, +``test_trigger_policy.py``). +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from leapflow.daemon.client import DaemonUnavailableError +from leapflow.daemon.lifecycle import DaemonInfo, cleanup_stale +from tests._harness.cassette_proxy import answer, scripted +from tests._harness.journey import Journey, JourneyFactory +from tests._harness.leapd import await_for, start_leapd + +# Journey metadata read by tools/impact.py (see test_r1_conversation.py). +SUBJECT_PATHS = ( + "src/leapflow/daemon/", + "src/leapflow/layout.py", + "src/leapflow/cli/commands/daemon.py", +) + +# No LLM semantics: process lifecycle, stale runtime files, and session resume. +# A live run would spend tokens for no extra signal. +LIVE_SIGNAL = False + +SESSION = "r6-lifecycle" + + +async def _turn(client: Any, message: str, workspace: str) -> list[Any]: + """Run one turn and return its stream events.""" + events: list[Any] = [] + async for event in client.engine_chat( + message, session_id=SESSION, workspace_root=workspace + ): + events.append(event) + return events + + +@pytest.mark.asyncio +async def test_r6_daemon_lifecycle(journeys: JourneyFactory) -> None: + """The daemon starts, reports itself, serves work, stops, and recovers cleanly.""" + journey = journeys( + "r6_lifecycle", + script=scripted(answer("Still here.")), + deadline_s=120.0, + # One turn; the rest is lifecycle. + max_llm_calls=6, + max_llm_tokens=80_000, + ) + workspace = str(journey.workspace("life")) + client = journey.client() + + with journey.phase("running: lifecycle artefacts exist and agree with each other"): + info = journey.daemon.info() + assert info.is_running, "the daemon reports itself as not running" + assert info.is_healthy, "the socket exists but is not answering" + assert journey.daemon.sock_path.exists(), "no Unix socket on disk" + + status = await client.status() + assert status["pid"] == info.pid, ( + f"status() reports pid {status['pid']} but the pid file says {info.pid}" + ) + + with journey.phase("identity: the daemon describes its own runtime, not the client's"): + status = await client.status() + assert status["runtime_version"], "the daemon reported no version" + assert status["runtime_executable"], "the daemon reported no executable" + assert str(journey.daemon.data_dir) in status["profile_dir"], ( + f"the daemon is serving {status['profile_dir']}, not this journey's " + f"profile under {journey.daemon.data_dir}" + ) + assert status["runtime_dir"] == str(journey.daemon.runtime_dir) + + with journey.phase("serving: a turn works, proving this is more than a socket"): + events = await _turn(client, "Are you there?", workspace) + assert not [event for event in events if event.type == "error"] + + with journey.phase("stop: shutdown removes the process and its runtime files"): + old_pid = journey.daemon.info().pid + try: + await client.shutdown() + except DaemonUnavailableError: + # A daemon that closes the socket while replying is a valid shutdown. + pass + gone = await await_for( + lambda: _not_running(journey), + timeout_s=30.0, + what="the daemon process to exit", + ) + assert gone, f"daemon pid {old_pid} is still running after shutdown" + + with journey.phase("stale state: leftover files do not block the next start"): + # Simulate the crash case: runtime files present, no process behind them. + journey.daemon.runtime_dir.mkdir(parents=True, exist_ok=True) + (journey.daemon.runtime_dir / "leapd.pid").write_text("999999", encoding="utf-8") + (journey.daemon.runtime_dir / "leapd.sock").touch(exist_ok=True) + + stale = DaemonInfo.discover(journey.daemon.runtime_dir) + assert not stale.is_healthy, "a stale socket was reported as healthy" + + removed = cleanup_stale(journey.daemon.runtime_dir) + assert removed, "stale runtime files were not cleaned up" + assert not (journey.daemon.runtime_dir / "leapd.pid").exists() + + with journey.phase("restart: a fresh daemon takes over the same profile"): + restarted = start_leapd( + root=journey.daemon.data_dir.parent, + llm_base_url=journey.proxy.base_url, + llm_model=journey.daemon.env["LEAPFLOW_LLM_MODEL"], + profile=journey.daemon.profile, + ) + journey.daemon.process = restarted.process + try: + assert restarted.info().is_healthy, "the replacement daemon never became healthy" + fresh_client = restarted.client() + status = await fresh_client.status() + assert status["pid"] != old_pid, ( + "the replacement daemon reports the dead process' pid" + ) + assert status["runtime_dir"] == str(journey.daemon.runtime_dir), ( + "the replacement daemon is not serving the same profile runtime" + ) + + with journey.phase("continuity: a prior session is resumable after restart"): + # A fresh daemon holds no live session, so history is only reachable + # the way a user reaches it: by resuming explicitly (`leap --resume`). + resumed = await fresh_client.session_resume(SESSION) + assert resumed.get("found") is True, ( + f"session {SESSION!r} was not recoverable after a restart: {resumed}" + ) + assert resumed.get("session_id") == SESSION, ( + f"resume returned a different session than asked for: {resumed}" + ) + history = await fresh_client.session_history(session_id=SESSION) + blob = str(history.get("messages") or []) + assert "Are you there?" in blob, ( + "the conversation recorded before the restart did not survive it" + ) + finally: + restarted.stop() + + journey.finish() + + +async def _not_running(journey: Journey) -> bool: + """True once no process owns the daemon's runtime directory.""" + return not DaemonInfo.discover(journey.daemon.runtime_dir).is_running diff --git a/tests/regression/__init__.py b/tests/regression/__init__.py new file mode 100644 index 0000000..40a4adb --- /dev/null +++ b/tests/regression/__init__.py @@ -0,0 +1 @@ +"""Regression ledger: one file per incident, always run, never selected away.""" diff --git a/tests/regression/test_impact_selection.py b/tests/regression/test_impact_selection.py new file mode 100644 index 0000000..ece241c --- /dev/null +++ b/tests/regression/test_impact_selection.py @@ -0,0 +1,235 @@ +"""Guards for change-scoped test selection. + +Selection decides which tests get a chance to fail, so a defect here silently +shrinks the suite — the exact failure mode the impact map exists to prevent. +These tests pin the properties that matter: shared foundations escalate to a full +run, a leaf change stays narrow, an unknown file falls back to the import graph +rather than being skipped, and the always-on tiers are never selected away. +""" + +from __future__ import annotations + +import pathlib +import sys + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from tools import impact # noqa: E402 + + +def test_escalation_file_exists_and_declares_foundations() -> None: + """The escalation list must exist and name the shared foundations. + + Without it every change would take the narrow path, including a change to + ``config.py`` — whose blast radius is the whole codebase. + """ + patterns = impact.escalation_patterns() + assert patterns, f"no escalation rules parsed from {impact.ESCALATE_FILE}" + required = ( + "src/leapflow/config.py", + "src/leapflow/layout.py", + "src/leapflow/engine/engine.py", + "src/leapflow/daemon/service.py", + "tests/conftest.py", + "pyproject.toml", + ) + missing = [rule for rule in required if rule not in patterns] + assert missing == [], f"escalation rules do not cover: {missing}" + + +@pytest.mark.parametrize( + "changed", + [ + "src/leapflow/config.py", + "src/leapflow/layout.py", + "src/leapflow/domain/events.py", + "src/leapflow/engine/engine.py", + "src/leapflow/daemon/session_registry.py", + "tests/conftest.py", + "tests/_harness/journey.py", + "pyproject.toml", + ".github/workflows/ci.yaml", + ], +) +def test_foundation_changes_force_a_full_run(changed: str) -> None: + """A change to any shared foundation selects nothing, meaning "run everything".""" + selected, reason = impact.select_from_paths([changed], coverage={}) + assert selected == [], f"{changed} should escalate but selected {selected}" + assert "full mock layer" in reason + assert changed in reason, f"the reason must name the file that escalated: {reason}" + + +def test_leaf_change_selects_only_related_tests_via_coverage_map() -> None: + """A coverage map hit narrows the run without falling back to everything.""" + coverage = { + "tests/test_web_fetch.py": ["src/leapflow/tools/web_fetch.py"], + "tests/test_repo_map.py": ["src/leapflow/tools/repo_map.py"], + } + selected, reason = impact.select_from_paths( + ["src/leapflow/tools/web_fetch.py"], coverage=coverage, patterns=[] + ) + assert selected == ["tests/test_web_fetch.py"], selected + assert "1 via coverage map" in reason + + +def test_unknown_source_file_falls_back_to_the_import_graph() -> None: + """A file the map has never seen must still select its dependents. + + A new module is precisely the case where the coverage map is empty, so + trusting the map alone would run nothing for brand-new code. + """ + target = "src/leapflow/llm/openai_provider.py" + assert (REPO_ROOT / target).is_file(), "fixture path no longer exists" + + selected, reason = impact.select_from_paths([target], coverage={}, patterns=[]) + assert selected, "an unknown source file selected no tests at all" + assert "via import graph" in reason + assert "1 source file(s) not in the map" in reason + + +def test_editing_a_test_file_always_runs_it() -> None: + """A directly edited test must run even when no source changed.""" + selected, _ = impact.select_from_paths( + ["tests/test_pure_algorithms.py"], coverage={}, patterns=[] + ) + assert "tests/test_pure_algorithms.py" in selected + + +def test_documentation_only_change_does_not_narrow_the_run() -> None: + """A change outside src/ and tests/ cannot be reasoned about, so run everything. + + Being conservative here costs one full run; being clever risks missing the + case where a doc change accompanied something else. + """ + selected, reason = impact.select_from_paths(["README.md"], coverage={}, patterns=[]) + assert selected == [] + assert "neither src/ nor tests/" in reason + + +def test_no_changes_runs_everything() -> None: + """An empty diff must not be read as "nothing to test".""" + selected, reason = impact.select_from_paths([], coverage={}, patterns=[]) + assert selected == [] + assert "no changes detected" in reason + + +def test_always_on_tiers_are_never_selected_away() -> None: + """The real layer and the ledger run regardless of what changed. + + This is the anti-seesaw guarantee: a suite that can be skipped will be + skipped, and then a change to one module breaks another with a green run. + """ + targets = impact.always_on_targets() + assert "tests/journeys" in targets, "the real journeys must always run" + assert "tests/regression" in targets, "the incident ledger must always run" + assert "tests/test_architecture_contracts.py" in targets + + +def test_selected_targets_exist_on_disk() -> None: + """Every selectable path resolves, so pytest cannot fail on a stale name.""" + for target in impact.always_on_targets(): + assert (REPO_ROOT / target).exists(), f"always-on target {target} does not exist" + + +# ── Live journey selection ──────────────────────────────────────── +# +# This selection *is* wired into CI, unlike the mock-layer one: each live journey +# costs real tokens and real minutes, so picking the wrong set is expensive in one +# direction and blind in the other. + + +def test_every_journey_declares_its_metadata() -> None: + """A journey must state which sources it exercises and whether live adds signal. + + A missing declaration defaults to "always run live", which is the safe + direction but also the expensive one — so it has to be a conscious choice + rather than an omission. + """ + metadata = impact.journey_metadata() + assert metadata, "no journeys found" + undeclared = [item.path for item in metadata if not item.subject_paths] + assert undeclared == [], ( + f"these journeys declare no SUBJECT_PATHS, so every live run pays for them: " + f"{undeclared}" + ) + + +def test_declared_subject_paths_exist() -> None: + """Subject declarations must point at real source paths. + + A renamed module leaves a dead prefix behind, and a dead prefix silently stops + matching — the journey would quietly drop out of live selection. + """ + stale: list[str] = [] + for item in impact.journey_metadata(): + for prefix in item.subject_paths: + if not (REPO_ROOT / prefix).exists(): + stale.append(f"{item.path} -> {prefix}") + assert stale == [], ( + f"these SUBJECT_PATHS no longer exist, so the journey has silently dropped " + f"out of live selection: {stale}" + ) + + +def test_scheduled_run_takes_every_live_capable_journey() -> None: + """With no diff, the live lane runs everything that has live value.""" + selected, reason = impact.select_journeys([], live_only=True) + live_capable = [item.path for item in impact.journey_metadata() if item.live_signal] + assert sorted(selected) == sorted(live_capable), selected + assert "all eligible journeys" in reason + + +def test_journeys_without_live_signal_are_never_selected_live() -> None: + """Control-plane and lifecycle journeys must not spend tokens. + + They assert config layering, the vault, and process lifecycle — none of which + a real provider can influence. R4 is excluded too: it asserts on injected + failures a forwarding mode cannot produce. + """ + excluded = {item.path for item in impact.journey_metadata() if not item.live_signal} + assert excluded, "expected some journeys to opt out of the live lane" + for paths in ([], ["src/leapflow/config.py"], ["src/leapflow/daemon/service.py"]): + selected, _ = impact.select_journeys(paths, live_only=True) + leaked = excluded.intersection(selected) + assert leaked == set(), f"{leaked} would spend tokens for no signal" + + +def test_change_scoped_live_selection_narrows_to_declared_subjects() -> None: + """A labelled pull request runs only the journeys the change could break.""" + selected, reason = impact.select_journeys( + ["src/leapflow/recording/recorder.py"], live_only=True, patterns=[] + ) + assert selected == ["tests/journeys/test_r5_learning.py"], selected + assert "1/3" in reason + + +def test_foundation_change_takes_every_live_journey() -> None: + """An escalating change cannot be narrowed — not even in the live lane.""" + selected, reason = impact.select_journeys(["src/leapflow/config.py"], live_only=True) + live_capable = [item.path for item in impact.journey_metadata() if item.live_signal] + assert sorted(selected) == sorted(live_capable) + assert "escalation rule" in reason + + +def test_unrelated_change_selects_no_live_journey() -> None: + """A change no journey claims must not trigger a paid run. + + The offline lanes still run every journey, so nothing goes unverified; this + only declines to pay a provider for a change none of them exercises. + """ + selected, reason = impact.select_journeys( + ["src/leapflow/gateway/adapters/feishu.py"], live_only=True, patterns=[] + ) + assert selected == [] + assert "no eligible journey declares a subject" in reason + + +def test_editing_a_journey_selects_it_live() -> None: + """Changing a journey's own assertions must exercise them against a provider.""" + target = "tests/journeys/test_r1_conversation.py" + selected, _ = impact.select_journeys([target], live_only=True, patterns=[]) + assert target in selected diff --git a/tests/regression/test_incident_ledger.py b/tests/regression/test_incident_ledger.py new file mode 100644 index 0000000..03d4abe --- /dev/null +++ b/tests/regression/test_incident_ledger.py @@ -0,0 +1,269 @@ +"""The incident ledger: one entry per outage that shipped with a green suite. + +Every entry below is a real regression that reached users. What they had in +common was that the suite agreed with the defect — so the ledger's job is not to +re-test the fix, but to make sure the coverage that now catches it cannot +disappear quietly. A ledger entry fails when its home is deleted, renamed, or +moved somewhere that change-scoped selection could skip. + +The ledger also carries structural guards for contracts that have no natural +home in a single module's tests, because that is exactly where these defects hid. +Where a behavioral guard exists, it is preferred: an end-to-end assertion that a +session-scoped status reports real usage is stronger and more durable than any +rule about which attribute the reporting code happens to read. + +Adding an entry is how a post-mortem ends. Removing one requires arguing that the +failure mode is now impossible, not merely unlikely. +""" + +from __future__ import annotations + +import ast +import pathlib +from dataclasses import dataclass + +import pytest + +TESTS_ROOT = pathlib.Path(__file__).resolve().parents[1] +SRC_ROOT = TESTS_ROOT.parent / "src" / "leapflow" + + +@dataclass(frozen=True) +class Incident: + """One past outage and the coverage that now catches it.""" + + key: str + symptom: str + home: str + required_tests: tuple[str, ...] + journeys: tuple[str, ...] = () + + +LEDGER: tuple[Incident, ...] = ( + Incident( + key="status-bar-frozen-at-zero", + symptom=( + "The status bar showed 0/ forever because reporting code read " + "ctx.engine — the template sessions are cloned from, which never " + "accumulates turns — instead of the caller's session engine." + ), + home="test_runtime_metadata_and_wrapping.py", + required_tests=(), + # The behavioral guard: a session-scoped status must report real usage, + # and an unscoped one must report none. That is a stronger check than any + # structural rule about which attribute the reporting code reads. + journeys=("test_r1_conversation.py", "test_r2_isolation.py"), + ), + Incident( + key="session-identity-adopted-from-another-client", + symptom=( + "A second TUI adopted the first client's session id from an unscoped " + "status(), sent it with its own workspace, and was rejected on every " + "turn — with advice that could not work, because a fresh client " + "re-adopted the same id on its first poll." + ), + home="test_multi_client_session_isolation.py", + required_tests=( + "test_status_without_a_session_reports_no_identity", + "test_client_adopts_a_session_only_when_it_has_none", + "test_cross_client_fallback_is_named_for_what_it_does", + "test_reusing_a_session_from_another_workspace_is_refused", + ), + journeys=("test_r2_isolation.py",), + ), + Incident( + key="local-defect-classified-as-provider-condition", + symptom=( + "A mistyped attribute whose name contained 'context' raised " + "AttributeError inside the provider call's try block; the message-" + "matching provider classifier read it as a context overflow and drove " + "every turn through three compressions, a failover, and a credential " + "rotation before halting." + ), + home="test_internal_defect_reporting.py", + required_tests=( + "test_defect_types_are_matched_by_type_not_message", + "test_internal_defect_is_not_classified_as_a_provider_condition", + "test_real_provider_conditions_still_use_the_provider_taxonomy", + "test_terminal_decision_carries_an_actionable_interaction", + ), + journeys=("test_r4_recovery.py",), + ), + Incident( + key="long-answers-lost-their-tail", + symptom=( + "prompt_toolkit's renderer clips at the window edge rather than " + "reflowing, so enabling soft_wrap on the shared console silently " + "truncated the end of every long answer." + ), + home="test_runtime_metadata_and_wrapping.py", + required_tests=(), + ), + Incident( + key="calibration-wiring-faked-by-tests", + symptom=( + "Calibration tests built the engine with object.__new__ and assigned " + "the attributes the method reads, so they agreed with a wrong " + "attribute name and stayed green while every real turn raised " + "AttributeError." + ), + home="test_budget_calibration.py", + required_tests=( + "test_real_engine_calibrates_on_the_production_path", + "test_telemetry_helper_absorbs_its_own_defects", + ), + ), +) + + +def _test_names(path: pathlib.Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + return { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + } + + +@pytest.mark.parametrize("incident", LEDGER, ids=lambda item: item.key) +def test_incident_coverage_still_exists(incident: Incident) -> None: + """Each recorded outage still has a home, with the tests that catch it. + + A deleted or renamed test is how a fixed defect becomes an unfixed one again. + """ + home = TESTS_ROOT / incident.home + assert home.is_file(), ( + f"incident {incident.key!r} has no coverage left: {incident.home} is gone.\n" + f"Symptom that will return: {incident.symptom}" + ) + if incident.required_tests: + present = _test_names(home) + missing = sorted(set(incident.required_tests) - present) + assert missing == [], ( + f"incident {incident.key!r} lost these guards from {incident.home}: {missing}\n" + f"Symptom that will return: {incident.symptom}" + ) + + for journey in incident.journeys: + path = TESTS_ROOT / "journeys" / journey + assert path.is_file(), ( + f"incident {incident.key!r} lost its end-to-end guard: " + f"journeys/{journey} is gone.\n" + f"Symptom that will return: {incident.symptom}" + ) + + +def test_ledger_covers_every_known_incident() -> None: + """The ledger is the index of past outages; keep it complete. + + AGENTS.md records five failures that shipped green. If the ledger holds fewer, + one of them has no permanent home. + """ + assert len(LEDGER) >= 5, f"the ledger has shrunk to {len(LEDGER)} entries" + assert len({item.key for item in LEDGER}) == len(LEDGER), "duplicate ledger keys" + for item in LEDGER: + assert item.symptom.strip(), f"{item.key} has no symptom description" + + +# ── Structural guards for contracts with no single natural home ─────────── + + +def _keyword_values(path: pathlib.Path, keyword: str) -> list[tuple[int, object]]: + """Return (line, value) for every ``keyword=`` argument in ``path``. + + Parsed rather than grepped: the incident that motivated this guard is + described in a comment inside the very file being checked, so a text search + finds the explanation and reports it as the defect. + """ + tree = ast.parse(path.read_text(encoding="utf-8")) + found: list[tuple[int, object]] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for kwarg in node.keywords: + if kwarg.arg == keyword and isinstance(kwarg.value, ast.Constant): + found.append((kwarg.lineno, kwarg.value.value)) + return found + + +def _reporting_modules() -> list[pathlib.Path]: + """Modules that report runtime state to a client.""" + candidates = [ + SRC_ROOT / "daemon" / "service.py", + SRC_ROOT / "daemon" / "session_coordinator.py", + ] + return [path for path in candidates if path.is_file()] + + +def test_session_engine_resolution_has_a_single_entry_point() -> None: + """The resolver that reporting code must use has to keep existing. + + A structural ban on reading ``ctx.engine`` is the wrong guard: the same + expression is legitimate when fetching the *template* to clone session engines + from, and telling those two uses apart from the syntax alone needs a brittle + rule. What can be checked cheaply is that the single sanctioned entry point + still exists — and the behavioral proof lives in R1/R2, which assert that a + session-scoped status reports real usage while an unscoped one reports none. + """ + names = {"resolve_session_engine": False, "_active_engine": False} + for path in _reporting_modules(): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name in names: + names[node.name] = True + + missing = sorted(name for name, found in names.items() if not found) + assert missing == [], ( + f"the sanctioned session-engine resolver(s) {missing} no longer exist; " + "without them reporting code resolves the engine template and silently " + "reports zeros, which froze the status bar at 0/" + ) + + +def test_cross_session_fallback_is_named_for_what_it_does() -> None: + """An aggregate resolver must say so in its name. + + "the current session" invites the misuse that leaked one client's identity to + another; ``most_recent_any_client`` cannot be mistaken for a per-caller lookup. + """ + registry = SRC_ROOT / "daemon" / "session_registry.py" + if not registry.is_file(): + pytest.skip("session registry has moved; update this ledger entry") + tree = ast.parse(registry.read_text(encoding="utf-8")) + method_names = { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + assert "most_recent_any_client" in method_names, ( + "the cross-session fallback was renamed; a resolver that ignores workspace " + "and client identity must keep saying so in its name" + ) + for banned in ("current_session", "get_current", "active_session"): + assert banned not in method_names, ( + f"{registry.name} defines {banned!r} — a friendly name for an aggregate " + "lookup is what let one client adopt another's session" + ) + + +def test_shared_console_does_not_enable_soft_wrap() -> None: + """Wrapping belongs to the console layer, with ``soft_wrap`` off. + + prompt_toolkit's renderer clips at the window edge instead of reflowing, so + ``soft_wrap=True`` on the shared console drops the tail of long answers. + """ + console_path = SRC_ROOT / "cli" / "tui_app" / "console.py" + if not console_path.is_file(): + pytest.skip("console module has moved; update this ledger entry") + + settings = _keyword_values(console_path, "soft_wrap") + enabled = [line for line, value in settings if value is True] + assert enabled == [], ( + f"{console_path.name} enables soft_wrap at line(s) {enabled}; long answers " + "will lose their tail under prompt_toolkit's renderer" + ) + assert any(value is False for _, value in settings), ( + f"{console_path.name} no longer states soft_wrap explicitly — the wrapping " + "contract must stay visible at the call site" + ) diff --git a/tests/regression/test_provider_shape_drift.py b/tests/regression/test_provider_shape_drift.py new file mode 100644 index 0000000..6bc9af2 --- /dev/null +++ b/tests/regression/test_provider_shape_drift.py @@ -0,0 +1,191 @@ +"""Provider response-shape drift guard. + +``tools/sync_fixtures.py`` distils every recorded cassette into the *shapes* the +provider actually sends. This guard asserts that those shapes still contain the +fields LeapFlow's parsers read. It is the drift detector the mock layer never +had: previously a provider could drop or rename a field and the suite would keep +passing, because every mock returned a body written from memory. + +When this fails, the fix is not to relax the assertion. It is to look at the +diff in ``response_shapes.json`` and update the parser that depended on the +field which disappeared. +""" + +from __future__ import annotations + +import json +import pathlib +from typing import Any + +import pytest + +FIXTURE = ( + pathlib.Path(__file__).resolve().parents[1] + / "_fixtures" + / "llm_responses" + / "response_shapes.json" +) +RECORDING_ROOT = pathlib.Path(__file__).resolve().parents[1] / "_fixtures" / "recordings" + +# Fields the production parser in leapflow/llm/openai_provider.py reads from a +# successful non-streamed completion. +_REQUIRED_COMPLETION_PATHS = ( + ("choices", 0, "message", "role"), + ("choices", 0, "message", "content"), + ("choices", 0, "finish_reason"), + ("usage", "prompt_tokens"), + ("usage", "completion_tokens"), + ("usage", "total_tokens"), +) + +# Fields read when the model asks for a tool. +_REQUIRED_TOOL_CALL_PATHS = ( + ("choices", 0, "message", "tool_calls", 0, "id"), + ("choices", 0, "message", "tool_calls", 0, "function", "name"), + ("choices", 0, "message", "tool_calls", 0, "function", "arguments"), +) + +# Fields the recovery classifier reads from an error body. +_REQUIRED_ERROR_PATHS = ( + ("error", "message"), + ("error", "type"), + ("error", "code"), +) + +# Usage keys the effective-cost accounting depends on. +_REQUIRED_USAGE_FIELDS = ("prompt_tokens", "completion_tokens", "total_tokens") + + +def _shapes() -> dict[str, Any]: + if not FIXTURE.is_file(): + pytest.skip(f"no derived fixtures at {FIXTURE}; run `make sync-fixtures`") + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def _resolve(shape: Any, path: tuple[Any, ...]) -> Any: + """Walk a derived shape by path, returning None when a step is absent.""" + node = shape + for step in path: + if isinstance(step, int): + if not isinstance(node, list) or not node: + return None + node = node[0] + continue + if not isinstance(node, dict) or step not in node: + return None + node = node[step] + return node + + +def _has(shapes: list[Any], path: tuple[Any, ...]) -> bool: + return any(_resolve(shape, path) is not None for shape in shapes) + + +def test_derived_fixtures_are_present_and_non_trivial() -> None: + """The distilled shapes must actually describe recorded traffic.""" + shapes = _shapes() + assert shapes.get("stored_responses_seen", 0) > 0, ( + "the derived fixture reports no recorded responses; re-run " + "`make seed-cassettes && make sync-fixtures`" + ) + assert shapes.get("completion_shapes"), "no successful completion shape recorded" + assert shapes.get("error_shapes"), ( + "no error shape recorded — the recovery classifier's inputs are unverified" + ) + + +def test_recorded_traffic_carries_the_optional_fields_production_reads() -> None: + """Real recordings must still expose the fields the provider layer opts into. + + These were the concrete proof that hand-written bodies drift: the seeded + cassettes carried neither ``reasoning_content`` (read to surface thinking on + dashscope/deepseek profiles) nor ``prompt_tokens_details`` (read by + ``_extract_cached_tokens`` for prefix-cache accounting). Both appeared only + once real traffic was recorded. Losing them again would silently disable two + production features. + """ + shapes = _shapes() + if not RECORDING_ROOT.is_dir() or not any(RECORDING_ROOT.iterdir()): + pytest.skip( + "no real recordings committed; these fields only appear in provider " + "traffic (run `make record-traffic` with credentials)" + ) + + completions = shapes.get("completion_shapes") or [] + assert _has(completions, ("choices", 0, "message", "reasoning_content")), ( + "no recorded response carries reasoning_content; thinking extraction is " + "now asserted only against bodies nobody has verified" + ) + reported = set(shapes.get("usage_fields") or ()) + assert "prompt_tokens_details" in reported, ( + "no recorded response reports prompt_tokens_details; prefix-cache " + "accounting in _extract_cached_tokens is unverified" + ) + + +def test_completion_shape_carries_every_field_the_parser_reads() -> None: + """A successful body must still expose the fields the provider layer reads.""" + completions = _shapes().get("completion_shapes") or [] + missing = [ + ".".join(str(step) for step in path) + for path in _REQUIRED_COMPLETION_PATHS + if not _has(completions, path) + ] + assert missing == [], ( + "recorded provider responses no longer carry these fields, which " + f"leapflow/llm/openai_provider.py reads: {missing}" + ) + + +def test_tool_call_shape_carries_every_field_the_engine_reads() -> None: + """Native tool calls must still expose id, name, and arguments.""" + completions = _shapes().get("completion_shapes") or [] + if not _has(completions, ("choices", 0, "message", "tool_calls")): + pytest.skip("no tool-calling response recorded yet") + missing = [ + ".".join(str(step) for step in path) + for path in _REQUIRED_TOOL_CALL_PATHS + if not _has(completions, path) + ] + assert missing == [], ( + f"recorded tool calls are missing fields the engine dispatches on: {missing}" + ) + + +def test_error_shape_carries_every_field_the_classifier_reads() -> None: + """Error bodies must still expose message, type, and code.""" + errors = _shapes().get("error_shapes") or [] + missing = [ + ".".join(str(step) for step in path) + for path in _REQUIRED_ERROR_PATHS + if not _has(errors, path) + ] + assert missing == [], ( + "recorded provider errors no longer carry these fields, which the " + f"recovery classifier reads: {missing}" + ) + + +def test_usage_accounting_fields_are_present() -> None: + """Token accounting depends on these keys; their loss is silent otherwise.""" + reported = set(_shapes().get("usage_fields") or ()) + missing = [field for field in _REQUIRED_USAGE_FIELDS if field not in reported] + assert missing == [], ( + f"providers no longer report {missing}; token accounting and the context " + "status bar read them" + ) + + +def test_recorded_error_codes_cover_the_recovery_categories() -> None: + """The failure classes the recovery journey depends on must be recorded. + + Without a recorded example of each, the classifier's handling of that class + is asserted only against a body somebody wrote by hand. + """ + codes = set(_shapes().get("error_codes") or ()) + expected = {"rate_limit_exceeded", "context_length_exceeded"} + missing = sorted(expected - codes) + assert missing == [], ( + f"no recorded provider error for {missing}; the recovery journey's " + "injected failures are what keep these classes honest" + ) diff --git a/tests/regression/test_suite_budget.py b/tests/regression/test_suite_budget.py new file mode 100644 index 0000000..9879a23 --- /dev/null +++ b/tests/regression/test_suite_budget.py @@ -0,0 +1,138 @@ +"""Budget guard for the real end-to-end layer. + +The real layer earns the right to run on *every* push — never skipped by impact +selection — only by staying small. That is the whole anti-seesaw mechanism: a +suite that can be skipped is a suite that will be skipped, and then a change to +one module breaks another with a green run. So the size of this layer is a hard, +executable constraint rather than a convention. + +When this test fails, the fix is to *merge* a journey into an existing one, not +to raise the ceiling. +""" + +from __future__ import annotations + +import ast +import pathlib + +import pytest + +TESTS_ROOT = pathlib.Path(__file__).resolve().parents[1] +JOURNEYS_DIR = TESTS_ROOT / "journeys" +HARNESS_DIR = TESTS_ROOT / "_harness" + +# Ceilings from the testing plan. Raising either needs a design decision, not a +# quick edit: every added journey is paid for on every push, by every developer. +MAX_JOURNEY_CASES = 8 +MAX_JOURNEY_MODULES = 8 + + +def _journey_modules() -> list[pathlib.Path]: + """Return journey test modules (excluding conftest).""" + if not JOURNEYS_DIR.is_dir(): + return [] + return sorted(p for p in JOURNEYS_DIR.glob("test_*.py")) + + +def _test_functions(path: pathlib.Path) -> list[str]: + """Return top-level test function names defined in ``path``.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + return [ + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + ] + + +def test_journey_case_count_within_budget() -> None: + """The real layer stays within its case ceiling.""" + per_module = {path.name: _test_functions(path) for path in _journey_modules()} + total = sum(len(names) for names in per_module.values()) + breakdown = "\n ".join( + f"{name}: {len(names)} ({', '.join(names)})" for name, names in sorted(per_module.items()) + ) + assert total <= MAX_JOURNEY_CASES, ( + f"real end-to-end layer has {total} cases, over the {MAX_JOURNEY_CASES} ceiling.\n" + f"Merge phases into an existing journey instead of adding a case:\n {breakdown}" + ) + assert len(per_module) <= MAX_JOURNEY_MODULES, ( + f"{len(per_module)} journey modules, over the {MAX_JOURNEY_MODULES} ceiling" + ) + + +def test_journeys_are_not_parameterized() -> None: + """One journey is one case. + + Parameterization is how a coarse layer silently becomes a fine one: the case + count multiplies without a single new ``def test_``, and the per-case budget + stops meaning anything. + """ + offenders: list[str] = [] + for path in _journey_modules(): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute): + continue + if node.attr != "parametrize": + continue + offenders.append(f"{path.name}:{node.lineno}") + assert offenders == [], ( + "journeys must not be parameterized — express variation as phases inside " + f"one journey: {offenders}" + ) + + +def test_every_journey_declares_a_deadline_and_finishes() -> None: + """Each journey must call ``finish()`` so its budget and misses are asserted. + + ``finish()`` is where "no cassette miss" and "within the time budget" are + checked. A journey that forgets it can pass while replaying nothing, which + would make the whole layer decorative. + """ + missing: list[str] = [] + for path in _journey_modules(): + source = path.read_text(encoding="utf-8") + if ".finish()" not in source: + missing.append(path.name) + assert missing == [], ( + f"these journeys never call journey.finish(), so cassette misses and the " + f"time budget go unchecked: {missing}" + ) + + +def test_every_journey_declares_both_cost_ceilings() -> None: + """A journey must bound its provider calls *and* its tokens. + + The two catch different failures. The call ceiling stops a turn that never + converges; the token ceiling stops prompt growth, which raises cost without + adding a single round. Either one alone leaves a way for the live lane to get + slower or more expensive without anything turning red. + """ + missing: list[str] = [] + for path in _journey_modules(): + source = path.read_text(encoding="utf-8") + for setting in ("max_llm_calls=", "max_llm_tokens="): + if setting not in source: + missing.append(f"{path.name} is missing {setting.rstrip('=')}") + assert missing == [], ( + "these journeys do not bound their own cost, so a regression in " + f"convergence or prompt size would go unnoticed: {missing}" + ) + + +def test_harness_is_not_collected_as_tests() -> None: + """Harness modules are infrastructure and must not define test functions.""" + offenders: list[str] = [] + for path in sorted(HARNESS_DIR.glob("*.py")): + for name in _test_functions(path): + offenders.append(f"{path.name}::{name}") + assert offenders == [], ( + f"move these out of tests/_harness/ — harness code must stay infrastructure: {offenders}" + ) + + +@pytest.mark.parametrize("required", ["cassette.py", "cassette_proxy.py", "leapd.py", "journey.py"]) +def test_harness_modules_present(required: str) -> None: + """The four harness pieces the journeys depend on exist.""" + assert (HARNESS_DIR / required).is_file(), f"missing harness module {required}" diff --git a/tests/regression/test_test_layer_contracts.py b/tests/regression/test_test_layer_contracts.py new file mode 100644 index 0000000..126375f --- /dev/null +++ b/tests/regression/test_test_layer_contracts.py @@ -0,0 +1,263 @@ +"""Fitness functions for the test suite itself. + +The mock layer is an asset — 1400-plus cases of branch coverage that no +end-to-end journey could afford. What makes it a liability is when a mock +encodes an assumption the product does not hold, because then it agrees with a +defect instead of catching it. These guards keep the mock layer honest without +rewriting it: + +1. faked construction (``object.__new__``) must be backed by a real-instance test; +2. LLM response bodies come from recorded traffic, not from hand-written literals; +3. mocks stay on process/network/OS boundaries and never replace internal logic. + +Existing debt is listed explicitly and the lists may only shrink: a stale entry +fails the test, so paying the debt down is rewarded and quietly adding to it is +not possible. +""" + +from __future__ import annotations + +import ast +import pathlib + +TESTS_ROOT = pathlib.Path(__file__).resolve().parents[1] + +# ── Guard 1: faked wiring needs real-instance cover ────────────────────── + +# ``object.__new__(X)`` plus assignment of the private attributes X reads cannot +# detect a wrong attribute *name* — the test simply agrees with the typo. That is +# how the calibration feature raised AttributeError on every real turn while the +# suite stayed green. The technique is still legitimate for pinning down ordering +# contracts, but only when the same file also builds the class for real and drives +# the production path. This guard enforces that pairing. +# +# Files here predate the rule and still lack real-instance cover. The list must +# only ever shrink: a stale entry fails the test, so paying the debt is rewarded +# and adding to it is not possible without an explicit edit. +_FAKED_WIRING_DEBT = frozenset( + { + "test_context_budget_scaling.py", + "test_daemon_isolation.py", + } +) + +# ── Guard 2: no hand-written LLM response bodies ────────────────────────── + +# Markers of a hand-authored OpenAI-shaped body. Real shapes come from +# tests/_fixtures/cassettes (see tools/sync_fixtures.py), so a provider changing +# its payload shows up as a fixture diff instead of passing forever against a +# body nobody has verified. +_RESPONSE_BODY_MARKERS = ("chat.completion", "chatcmpl-", "prompt_tokens_details") + +_HAND_WRITTEN_BODY_ALLOWLIST = frozenset( + { + # The harness authors bodies on purpose: it is the component that defines + # what a recorded response looks like. + "_harness/cassette.py", + # The drift guard names the fields production reads so it can assert that + # recorded traffic still carries them. Naming a field is the opposite of + # hand-writing a body — it is what makes a missing field fail. + "regression/test_provider_shape_drift.py", + } +) + +# Predates the cassette store. Must only shrink — a stale entry fails the test. +_HAND_WRITTEN_BODY_DEBT = frozenset( + { + "test_adaptive_depth.py", + "test_gateway_adapters.py", + } +) + +# ── Guard 3: mocks only at boundaries ───────────────────────────────────── + +# Patch targets that replace LeapFlow's own decision-making rather than an +# external boundary. Mocking these proves the test's own arrangement, not the +# behavior under test. +_INTERNAL_PATCH_TARGETS = ( + "leapflow.engine.recovery_coordinator.RecoveryCoordinator.evaluate", + "leapflow.engine.unified_classifier", + "leapflow.daemon.session_registry.SessionRegistry.acquire", + "leapflow.config_service.ConfigService.set", +) + + +def _test_modules() -> list[pathlib.Path]: + """Return every test module, excluding this file.""" + return sorted( + path + for path in TESTS_ROOT.rglob("*.py") + if path.name.startswith(("test_", "conftest")) + and path.resolve() != pathlib.Path(__file__).resolve() + ) + + +def _relative(path: pathlib.Path) -> str: + return str(path.relative_to(TESTS_ROOT)) + + +def _harness_modules() -> list[pathlib.Path]: + """Return harness modules, which are infrastructure rather than tests.""" + return sorted((TESTS_ROOT / "_harness").glob("*.py")) + + +def test_faked_wiring_is_backed_by_a_real_instance_test() -> None: + """A file that fakes construction must also build the class for real. + + Faking is acceptable for ordering contracts, but on its own it can only + confirm the names the test itself chose. Constructing the same class properly + somewhere in the file is what makes a mistyped attribute fail. + """ + offenders: list[str] = [] + paid_off: list[str] = [] + for path in _test_modules(): + relative = _relative(path) + source = path.read_text(encoding="utf-8") + tree = ast.parse(source) + faked: dict[str, int] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute) or node.attr != "__new__": + continue + if not (isinstance(node.value, ast.Name) and node.value.id == "object"): + continue + parent_call = _enclosing_call(tree, node) + target = _first_name_argument(parent_call) + if target: + faked.setdefault(target, node.lineno) + + uncovered = [ + f"{relative}:{lineno} fakes {name} but never constructs {name}(...)" + for name, lineno in sorted(faked.items()) + if f"{name}(" not in source.replace(f"object.__new__({name})", "") + ] + if relative in _FAKED_WIRING_DEBT: + if not uncovered: + paid_off.append(relative) + continue + offenders.extend(uncovered) + + assert offenders == [], ( + "these files fake construction without ever driving the real one, so a " + "wrong attribute name cannot fail them — add a test that builds the class " + "and calls the production path:\n " + "\n ".join(offenders) + ) + assert paid_off == [], ( + "these files now have real-instance cover; remove them from " + f"_FAKED_WIRING_DEBT so the list keeps shrinking: {paid_off}" + ) + + +def _enclosing_call(tree: ast.AST, needle: ast.AST) -> ast.Call | None: + """Return the Call node whose func is ``needle``.""" + for node in ast.walk(tree): + if isinstance(node, ast.Call) and node.func is needle: + return node + return None + + +def _first_name_argument(call: ast.Call | None) -> str: + """Return the first positional argument's name, when it is a bare name.""" + if call is None or not call.args: + return "" + first = call.args[0] + return first.id if isinstance(first, ast.Name) else "" + + +def test_llm_response_bodies_come_from_recorded_traffic() -> None: + """Provider payload shapes are recorded, never hand-written. + + A hand-authored body freezes one developer's belief about the wire format. It + keeps passing after the provider changes, which is precisely the failure mode + the cassette store exists to remove. + """ + offenders: list[str] = [] + paid_off: list[str] = [] + for path in list(_test_modules()) + _harness_modules(): + relative = _relative(path) + if relative in _HAND_WRITTEN_BODY_ALLOWLIST: + continue + source = path.read_text(encoding="utf-8") + found = next((m for m in _RESPONSE_BODY_MARKERS if m in source), "") + if relative in _HAND_WRITTEN_BODY_DEBT: + if not found: + paid_off.append(relative) + continue + if found: + offenders.append(f"{relative} (contains {found!r})") + + assert offenders == [], ( + "these files hand-write an OpenAI-shaped response body; use a recorded " + "cassette or a fixture derived from one (make sync-fixtures):\n " + + "\n ".join(offenders) + ) + assert paid_off == [], ( + "these files no longer hand-write response bodies; remove them from " + f"_HAND_WRITTEN_BODY_DEBT: {paid_off}" + ) + + +def test_mocks_stay_on_external_boundaries() -> None: + """Mock external IO, never LeapFlow's own decision points. + + Replacing the recovery coordinator, the classifier, or the session registry + with a double means the test asserts its own stub. These are exactly the + components whose real behavior the suite is supposed to pin down. + """ + offenders: list[str] = [] + for path in _test_modules(): + source = path.read_text(encoding="utf-8") + for target in _INTERNAL_PATCH_TARGETS: + if f'"{target}"' in source or f"'{target}'" in source: + offenders.append(f"{_relative(path)} patches {target}") + + assert offenders == [], ( + "these tests replace internal logic with a double; mock only process, " + "network, or OS boundaries:\n " + "\n ".join(offenders) + ) + + +def test_journeys_do_not_mock_anything() -> None: + """The real layer earns its cost only by staying real. + + A journey that reaches for ``unittest.mock`` has stopped being an end-to-end + check and has become an expensive unit test: it pays for a daemon subprocess + and then stubs out the thing it was meant to exercise. + """ + journeys_dir = TESTS_ROOT / "journeys" + offenders: list[str] = [] + for path in sorted(journeys_dir.rglob("*.py")): + source = path.read_text(encoding="utf-8") + for marker in ("unittest.mock", "MagicMock", "AsyncMock", "monkeypatch"): + if marker in source: + offenders.append(f"{_relative(path)} uses {marker}") + + assert offenders == [], ( + "journeys must exercise the real system end to end; move anything that " + "needs a double down to the mock layer:\n " + "\n ".join(offenders) + ) + + +def test_committed_cassettes_are_readable_and_non_empty() -> None: + """Replay-lane inputs must be present and loadable. + + The PR and main lanes run offline against these files. A corrupt or missing + store turns every journey into a cassette-miss failure, so it is worth + failing fast with a clear reason. + """ + from tests._harness.cassette import CassetteStore + + root = TESTS_ROOT / "_fixtures" / "cassettes" + assert root.is_dir(), ( + f"no committed cassette store at {root} — run `make seed-cassettes`" + ) + journeys = sorted(p for p in root.iterdir() if p.is_dir()) + assert journeys, f"cassette store {root} has no journey directories" + + for directory in journeys: + store = CassetteStore(directory) # raises on unreadable content + assert len(store) > 0, f"cassette directory {directory.name} is empty" + for key in store.keys(): + record = store.get(key) + assert record is not None and record.responses, ( + f"cassette {key} in {directory.name} carries no response" + ) diff --git a/tests/test_journey_harness.py b/tests/test_journey_harness.py new file mode 100644 index 0000000..7eb415b --- /dev/null +++ b/tests/test_journey_harness.py @@ -0,0 +1,689 @@ +"""Tests for the end-to-end harness itself. + +The harness is the foundation the whole real layer stands on, so it gets the +same scrutiny as production code. In particular the proxy is exercised through +the *real* ``openai`` client here, because the point of the design is that the +SDK, httpx and SSE parsing stay in the path — a harness verified only through +direct method calls would prove nothing about that. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from tests._harness.cassette import ( + CassetteRecord, + CassetteResponse, + CassetteStore, + context_overflow_response, + fingerprint, + normalize_request, + rate_limited_response, + record_for, + scrub, + streamed_response, + truncated_stream_response, +) +from tests._harness.cassette_proxy import ( + REPLAY, + SEED, + CassetteProxy, + Script, + resolve_mode, +) +from tests._harness.leapd import hermetic_env + +pytestmark = pytest.mark.component + + +def _chat_body(prompt: str, *, stream: bool = True, model: str = "cassette-model") -> dict: + return { + "model": model, + "stream": stream, + "messages": [ + {"role": "system", "content": "You are LeapFlow."}, + {"role": "user", "content": prompt}, + ], + } + + +# ── Fingerprinting ─────────────────────────────────────────────────────── + + +def test_scrub_removes_volatile_substrings() -> None: + """Timestamps, ids and temp paths must not enter a fingerprint.""" + raw = ( + "at 2026-08-06T11:12:13Z session sess-9f3a2b1c in /var/folders/xy/T/pytest-1/ws " + "req 550e8400-e29b-41d4-a716-446655440000 epoch 1785000000 host 127.0.0.1:54321" + ) + cleaned = scrub(raw) + assert "2026-08-06" not in cleaned + assert "sess-9f3a2b1c" not in cleaned + assert "550e8400" not in cleaned + assert "1785000000" not in cleaned + assert "54321" not in cleaned + assert "" in cleaned and "" in cleaned and "" in cleaned + + +def test_fingerprint_is_stable_across_runs_but_sensitive_to_intent() -> None: + """Equivalent prompts match; a different question does not.""" + first = _chat_body("list files in /var/folders/ab/T/pytest-1/ws at 2026-08-06T00:00:00Z") + second = _chat_body("list files in /var/folders/zz/T/pytest-9/ws at 2026-08-07T10:30:00Z") + third = _chat_body("delete every file") + + assert fingerprint(first) == fingerprint(second) + assert fingerprint(first) != fingerprint(third) + + +def test_tool_result_identifiers_do_not_break_fingerprint_stability() -> None: + """A prompt carrying a tool result must fingerprint the same on every run. + + Tool results embed a fresh ``execution_id`` per call — an *undashed* 32-hex + id that the dashed-UUID rule does not match. Without scrubbing it, a single + tool use makes the follow-up prompt unique forever, so no tool-using journey + could ever replay and the offline lanes would be permanently red. + """ + def _with_execution_id(execution_id: str) -> dict: + result = { + "ok": True, + "path": "/tmp/x/invoice.txt", + "execution_id": execution_id, + "tool_call_id": "call_721d22c0479f40668a3aeddb", + "execution_policy": "read_only", + } + return { + "model": "m", + "stream": False, + "messages": [ + {"role": "user", "content": "read the invoice"}, + {"role": "tool", "tool_call_id": "call_1", "content": json.dumps(result)}, + ], + } + + first = _with_execution_id("9376486cbab04bfbaea595c5dd7a8d59a") + second = _with_execution_id("955e2afa50bf4530943346dc01f27122") + + assert fingerprint(first) == fingerprint(second), ( + "a per-call execution id leaked into the fingerprint" + ) + + +def test_short_hex_content_is_not_scrubbed() -> None: + """Scrubbing must not swallow ordinary content that merely looks like hex.""" + cleaned = scrub("the colour is deadbeef and the short sha is abc123") + assert "deadbeef" in cleaned + assert "abc123" in cleaned + + +def test_fingerprint_ignores_tool_schema_churn_but_tracks_tool_set() -> None: + """Descriptions change constantly; the available tool set is what matters.""" + base = _chat_body("hi") + with_v1 = { + **base, + "tools": [ + {"type": "function", "function": {"name": "read_file", "description": "Read a file"}} + ], + } + with_v2 = { + **base, + "tools": [ + { + "type": "function", + "function": {"name": "read_file", "description": "Read a file from disk (v2)"}, + } + ], + } + with_extra = { + **base, + "tools": [ + {"type": "function", "function": {"name": "read_file"}}, + {"type": "function", "function": {"name": "web_fetch"}}, + ], + } + + assert fingerprint(with_v1) == fingerprint(with_v2) + assert fingerprint(with_v1) != fingerprint(with_extra) + + +def test_stream_flag_and_model_separate_fingerprints() -> None: + """Streaming and non-streaming responses are different recordings.""" + assert fingerprint(_chat_body("hi")) != fingerprint(_chat_body("hi", stream=False)) + assert fingerprint(_chat_body("hi")) != fingerprint(_chat_body("hi", model="other")) + + +def test_multimodal_content_reduces_to_part_kinds() -> None: + """Image bytes are unstable; their presence still shapes the request.""" + payload = { + "model": "m", + "stream": False, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this at 2026-08-06T00:00:00Z"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], + } + ], + } + normalized = normalize_request(payload) + parts = normalized["messages"][0]["content"] + assert parts[0] == {"type": "text", "text": "what is this at "} + assert parts[1] == {"type": "image_url"} + + +# ── Store ──────────────────────────────────────────────────────────────── + + +def test_cassette_round_trips_through_disk(tmp_path: Path) -> None: + """A stored cassette reloads byte-identically, streams included.""" + store = CassetteStore(tmp_path) + body = _chat_body("hello") + record = record_for(body, streamed_response("he", "llo"), note="greeting") + store.put(record) + + reloaded = CassetteStore(tmp_path) + restored = reloaded.get(fingerprint(body)) + assert restored is not None + assert restored.note == "greeting" + assert restored.responses[0].frames == record.responses[0].frames + assert restored.responses[0].content_type == "text/event-stream" + + +def test_response_sequence_is_consumed_in_order_then_repeats(tmp_path: Path) -> None: + """Retry paths resend an identical request and must see different answers.""" + store = CassetteStore(tmp_path) + body = _chat_body("hello") + store.put(record_for(body, rate_limited_response(), streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY) + + first = proxy.handle_chat(body) + second = proxy.handle_chat(body) + third = proxy.handle_chat(body) + + assert first.status == 429 + assert second.status == 200 and second.is_stream + assert third.status == 200, "the last response repeats once the sequence is exhausted" + + +def test_explain_miss_diffs_against_the_nearest_request(tmp_path: Path) -> None: + """A miss must name what drifted, not just report a hash.""" + store = CassetteStore(tmp_path) + store.put(record_for(_chat_body("summarize the report"), streamed_response("ok"))) + + explanation = store.explain_miss(_chat_body("summarize the invoice")) + + assert "no cassette for fingerprint" in explanation + assert "Nearest stored request" in explanation + assert "invoice" in explanation + + +def test_explain_miss_on_empty_store_points_at_recording(tmp_path: Path) -> None: + """An empty store is a setup problem and must say so.""" + explanation = CassetteStore(tmp_path).explain_miss(_chat_body("hi")) + assert "make seed-cassettes" in explanation + + +def test_binary_bodies_survive_via_base64(tmp_path: Path) -> None: + """Non-UTF8 payloads must round-trip losslessly.""" + store = CassetteStore(tmp_path) + raw = b"\xff\xfe\x00binary" + store.put( + CassetteRecord( + fingerprint="deadbeef", + request={"model": "m"}, + responses=(CassetteResponse(status=200, body=raw),), + ) + ) + restored = CassetteStore(tmp_path).get("deadbeef") + assert restored is not None + assert restored.responses[0].body == raw + + +# ── Proxy over real HTTP, driven by the real OpenAI SDK ─────────────────── + + +@pytest.mark.asyncio +async def test_replay_streams_through_the_real_openai_client(tmp_path: Path) -> None: + """The recorded SSE frames must parse through the production provider. + + This is the whole point of the proxy: ``OpenAIChat`` — not a stub — reads the + stream, so frame-level and usage-parsing defects are caught here. + """ + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + messages = [ + {"role": "system", "content": "You are LeapFlow."}, + {"role": "user", "content": "say hello"}, + ] + body = {"model": "cassette-model", "stream": True, "messages": messages} + store.put(record_for(body, streamed_response("Hel", "lo!"))) + + with CassetteProxy(store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + response = await provider.achat(messages, stream=True) + + assert response.content == "Hello!" + assert response.usage.get("total_tokens") == 80 + proxy.assert_no_misses() + assert proxy.stats.call_count == 1 + + +@pytest.mark.asyncio +async def test_injected_rate_limit_is_retried_by_the_provider(tmp_path: Path) -> None: + """A 429 recording must drive the real retry path, not a simulated one.""" + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + messages = [{"role": "user", "content": "retry me"}] + body = {"model": "cassette-model", "stream": True, "messages": messages} + store.put(record_for(body, rate_limited_response(), streamed_response("recovered"))) + + with CassetteProxy(store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=3, + ) + response = await provider.achat(messages, stream=True) + + assert response.content == "recovered" + assert proxy.stats.call_count == 2, "the provider must have retried the 429" + + +@pytest.mark.asyncio +async def test_context_overflow_surfaces_as_a_provider_error(tmp_path: Path) -> None: + """A 400 context-length body must reach the caller as an error, not empty text.""" + import openai + + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + messages = [{"role": "user", "content": "way too long"}] + body = {"model": "cassette-model", "stream": True, "messages": messages} + store.put(record_for(body, context_overflow_response())) + + with CassetteProxy(store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + with pytest.raises(openai.APIStatusError) as caught: + await provider.achat(messages, stream=True) + + assert "maximum context length" in str(caught.value) + assert proxy.stats.call_count == 1, "a 400 must not be retried" + + +@pytest.mark.asyncio +async def test_truncated_stream_does_not_silently_yield_a_partial_answer( + tmp_path: Path, +) -> None: + """A dropped stream must be observable, not rendered as a complete reply.""" + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + messages = [{"role": "user", "content": "long answer"}] + body = {"model": "cassette-model", "stream": True, "messages": messages} + store.put(record_for(body, truncated_stream_response("par", "tial"))) + + with CassetteProxy(store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + response = await provider.achat(messages, stream=True) + + # The content that did arrive is preserved, but no finish_reason was sent — + # which is how a caller can tell the turn was cut short. + assert response.content == "partial" + assert not response.finish_reason + + +@pytest.mark.asyncio +async def test_replay_miss_fails_loudly_with_a_diff(tmp_path: Path) -> None: + """An unmatched request must fail the test, never fall through to silence.""" + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + store.put(record_for(_chat_body("known question"), streamed_response("ok"))) + + with CassetteProxy(store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + with pytest.raises(Exception): + await provider.achat([{"role": "user", "content": "unknown question"}], stream=True) + + with pytest.raises(AssertionError) as caught: + proxy.assert_no_misses() + + assert "cassette miss" in str(caught.value) + assert "Nearest stored request" in str(caught.value) + + +@pytest.mark.asyncio +async def test_seed_mode_persists_scripted_exchanges_as_cassettes(tmp_path: Path) -> None: + """Seeding produces a committed store that later replays offline.""" + from leapflow.llm.openai_provider import OpenAIChat + + store = CassetteStore(tmp_path) + messages = [{"role": "user", "content": "seed me"}] + + with CassetteProxy(store, mode=SEED, script=Script.of("seeded answer")) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + seeded = await provider.achat(messages, stream=True) + proxy.assert_no_misses() + + assert seeded.content == "seeded answer" + + replayed_store = CassetteStore(tmp_path) + assert len(replayed_store) == 1 + with CassetteProxy(replayed_store, mode=REPLAY) as proxy: + provider = OpenAIChat( + api_key="cassette-key", + base_url=proxy.base_url, + model="cassette-model", + max_retries=1, + ) + replayed = await provider.achat(messages, stream=True) + proxy.assert_no_misses() + + assert replayed.content == "seeded answer" + + +def test_proxy_exposes_prompt_traffic_for_assertions(tmp_path: Path) -> None: + """Journeys assert on what reached the model, e.g. that a tool result fed back.""" + store = CassetteStore(tmp_path) + body = _chat_body("check the invoice total") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY) + + proxy.handle_chat(body) + + assert proxy.stats.call_count == 1 + assert proxy.stats.prompts_containing("invoice total") + assert not proxy.stats.prompts_containing("purchase order") + + +def test_forwarding_mode_requires_an_upstream(tmp_path: Path) -> None: + """Record/live without an upstream is a configuration error, caught at build.""" + with pytest.raises(ValueError, match="upstream base URL"): + CassetteProxy(CassetteStore(tmp_path), mode="record") + + +def test_unknown_mode_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + """A typo in the mode env var must fail immediately, not default silently.""" + monkeypatch.setenv("LEAPFLOW_TEST_LLM_MODE", "reply") + with pytest.raises(ValueError, match="not one of"): + resolve_mode() + + +# ── Convergence: the provider-call ceiling ────────────────────────────── + + +def test_call_budget_cuts_off_a_non_converging_loop(tmp_path: Path) -> None: + """A turn that keeps asking the model must be stopped at the ceiling. + + This is the guard against the failure the whole budget exists for: a loop + that never converges would otherwise run to the engine's iteration cap, + costing minutes offline and real money live. The ceiling has to be enforced + at the boundary, not merely asserted after the fact. + """ + store = CassetteStore(tmp_path) + body = _chat_body("loop forever") + store.put(record_for(body, streamed_response("again"))) + proxy = CassetteProxy(store, mode=REPLAY, max_calls=3) + + statuses = [proxy.handle_chat(body).status for _ in range(6)] + + assert statuses[:3] == [200, 200, 200], f"the budget bit too early: {statuses}" + assert statuses[3:] == [400, 400, 400], f"calls past the ceiling were served: {statuses}" + assert proxy.stats.budget_exceeded is True + + +def test_budget_refusal_is_not_retryable(tmp_path: Path) -> None: + """The refusal must halt the loop, not feed the provider's retry logic. + + A 429 or 5xx would be retried, so the runaway turn would keep going and burn + the retry budget on top of the call budget. + """ + store = CassetteStore(tmp_path) + body = _chat_body("one call only") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_calls=1) + + proxy.handle_chat(body) + refusal = proxy.handle_chat(body) + + assert refusal.status == 400, "a retryable status would let the loop continue" + assert b"journey_call_budget_exceeded" in refusal.body + + +def test_zero_budget_means_unlimited(tmp_path: Path) -> None: + """An unset ceiling must not accidentally block every call.""" + store = CassetteStore(tmp_path) + body = _chat_body("unbounded") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_calls=0) + + statuses = {proxy.handle_chat(body).status for _ in range(20)} + + assert statuses == {200} + assert proxy.stats.budget_exceeded is False + + +def test_token_budget_cuts_off_prompt_growth(tmp_path: Path) -> None: + """Cost must be capped by tokens too, not only by call count. + + This is the gap the call ceiling cannot close: a longer system prompt or a + bigger tool catalogue raises the bill without adding a single round, so a + call-count-only gate would let the live lane get quietly more expensive. + """ + store = CassetteStore(tmp_path) + body = _chat_body("expensive") + # streamed_response reports 80 total tokens per response. + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_calls=0, max_tokens=200) + + statuses = [proxy.handle_chat(body).status for _ in range(5)] + + assert statuses[:3] == [200, 200, 200], f"budget bit too early: {statuses}" + assert statuses[3:] == [400, 400], f"calls past the token ceiling served: {statuses}" + assert proxy.stats.token_budget_exceeded is True + assert proxy.stats.budget_exceeded is False, "the call ceiling must not be blamed" + + +def test_token_accounting_reads_streamed_and_whole_body_usage(tmp_path: Path) -> None: + """Usage must be counted for both wire forms the engine provokes. + + The native-tool round is non-streaming and a plain answer round streams, so + missing either form would under-count and silently disable the ceiling. + """ + from tests._harness.cassette import json_response, total_tokens_of + + assert total_tokens_of(streamed_response("a", "b")) == 80 + assert total_tokens_of(json_response(content="hi")) == 80 + assert total_tokens_of(rate_limited_response()) == 0, "an error body reports no usage" + + store = CassetteStore(tmp_path) + streaming = _chat_body("stream me", stream=True) + whole = _chat_body("whole body", stream=False) + store.put(record_for(streaming, streamed_response("ok"))) + store.put(record_for(whole, json_response(content="ok"))) + proxy = CassetteProxy(store, mode=REPLAY) + + proxy.handle_chat(streaming) + proxy.handle_chat(whole) + + assert proxy.stats.total_tokens == 160 + + +def test_token_budget_refusal_is_not_retryable(tmp_path: Path) -> None: + """The refusal must halt the loop rather than feed provider retries.""" + store = CassetteStore(tmp_path) + body = _chat_body("one only") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_tokens=1) + + first = proxy.handle_chat(body) + refusal = proxy.handle_chat(body) + + assert first.status == 200, "the first call must be served; the ceiling is cumulative" + assert refusal.status == 400 + assert b"journey_token_budget_exceeded" in refusal.body + + +def test_journey_finish_distinguishes_token_from_call_exhaustion(tmp_path: Path) -> None: + """The failure must name prompt growth, not blame a loop that did not happen.""" + from tests._harness.journey import Journey + + store = CassetteStore(tmp_path) + body = _chat_body("grow") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_tokens=1) + proxy.handle_chat(body) + proxy.handle_chat(body) + + journey = Journey( + journey_id="token-probe", + proxy=proxy, + daemon=_LogOnlyDaemon(), + max_llm_calls=99, + max_llm_tokens=1, + ) + with pytest.raises(AssertionError, match="prompt growth, not a loop"): + journey.finish() + + +def test_journey_finish_reports_an_exhausted_budget(tmp_path: Path) -> None: + """``finish()`` must name the budget as the cause, not just fail somewhere.""" + from tests._harness.journey import Journey + + store = CassetteStore(tmp_path) + body = _chat_body("loop") + store.put(record_for(body, streamed_response("ok"))) + proxy = CassetteProxy(store, mode=REPLAY, max_calls=1) + proxy.handle_chat(body) + proxy.handle_chat(body) + + journey = Journey( + journey_id="budget-probe", + proxy=proxy, + daemon=_LogOnlyDaemon(), + max_llm_calls=1, + ) + with pytest.raises(AssertionError, match="provider-call budget"): + journey.finish() + + +class _LogOnlyDaemon: + """Minimal stand-in for the daemon handle a Journey holds. + + Only ``tail_log`` is reachable from the paths under test here; starting a real + subprocess to assert a budget message would be the kind of cost the budget + itself exists to avoid. + """ + + def tail_log(self, limit: int = 60) -> str: + """Return an empty log; no daemon was started for this probe.""" + return "(no daemon)" + + +# ── Mode awareness ─────────────────────────────────────────────── + + +def test_forwarding_modes_ignore_the_cassette_store(tmp_path: Path) -> None: + """Live and record must reach the provider even when a recording exists. + + Serving a stored answer in a forwarding mode would make the live lane assert + against recordings — exactly the drift it is there to detect. It is also why + a journey asserting on injected failures cannot run live: the injection lives + in the store, which these modes bypass. + """ + from tests._harness.cassette_proxy import _FORWARD_MODES + + assert set(_FORWARD_MODES) == {"record", "live"} + + store = CassetteStore(tmp_path) + body = _chat_body("stored") + store.put(record_for(body, streamed_response("from the store"))) + + proxy = CassetteProxy( + store, + mode="live", + upstream_base_url="http://127.0.0.1:1/v1", + upstream_api_key="k", + ) + # Port 1 refuses connections, so a forwarded call raises rather than quietly + # returning the stored response. + with pytest.raises(Exception): + proxy.handle_chat(body) + + +# ── Daemon environment hermeticity ─────────────────────────────────────── + + +def test_hermetic_env_drops_inherited_leapflow_variables( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A developer's real credentials and data dir must never reach the daemon. + + Without this, a journey would read and write the user's actual profile and + could spend real tokens while claiming to run offline. + """ + monkeypatch.setenv("LEAPFLOW_LLM_API_KEY", "sk-real-user-key") + monkeypatch.setenv("LEAPFLOW_DATA_DIR", str(Path.home() / ".leapflow")) + monkeypatch.setenv("LEAPFLOW_GATEWAY_MANIFEST", "/somewhere/real.yaml") + monkeypatch.setenv("PATH", os.environ.get("PATH", "")) + + env = hermetic_env( + data_dir=tmp_path / "data", + profile="default", + llm_base_url="http://127.0.0.1:1/v1", + ) + + assert env["LEAPFLOW_LLM_API_KEY"] == "cassette-key" + assert env["LEAPFLOW_DATA_DIR"] == str(tmp_path / "data") + assert "LEAPFLOW_GATEWAY_MANIFEST" not in env, "inherited LEAPFLOW_* must be stripped" + assert env["PATH"], "non-LeapFlow environment must be preserved" + + +def test_hermetic_env_points_every_provider_at_the_proxy(tmp_path: Path) -> None: + """Aux and VLM clients must not be able to reach the network independently.""" + env = hermetic_env( + data_dir=tmp_path, + profile="default", + llm_base_url="http://127.0.0.1:9/v1", + ) + for key in ("LEAPFLOW_LLM_BASE_URL", "LEAPFLOW_LLM_AUX_BASE_URL", "LEAPFLOW_VLM_BASE_URL"): + assert env[key] == "http://127.0.0.1:9/v1" + + +def test_hermetic_env_forces_mock_host(tmp_path: Path) -> None: + """OS-host mocking is the one legitimate mock: CI has no macOS perception.""" + env = hermetic_env(data_dir=tmp_path, profile="default", llm_base_url="http://x/v1") + assert env["LEAPFLOW_MOCK_HOST"] == "1" diff --git a/tools/impact.py b/tools/impact.py new file mode 100644 index 0000000..51fc76d --- /dev/null +++ b/tools/impact.py @@ -0,0 +1,525 @@ +"""Change-scoped test selection. + +Two separate jobs, with different economics: + +**Mock layer** (``select_from_paths``) — available but *not wired into CI*. The +always-on tiers (real journeys, regression ledger, architecture contracts) can +never be selected away and already account for ~14s of an ~18s full run, so +selecting the mock layer can save at most a few seconds however precise it gets. +Kept for local use (``make test-impact``), where a single-module change narrows +to 2-3 test files, and for when the suite outgrows its feedback budget. + +**Live journeys** (``select_journeys``) — wired in, because here each journey costs +real tokens and real minutes against a real provider, so the arithmetic comes out +the other way. Journeys declare their own ``SUBJECT_PATHS`` and ``LIVE_SIGNAL``, so +the selection lives next to the assertions it describes. The *offline* journey +lanes never select: replay is cheap and a suite that can be skipped will be +skipped. + +Selection sources for the mock layer, in order of precedence: + +1. **Escalation rules** (``tests/.impact/escalate.yaml``) — a change to shared + foundations means everything runs. No cleverness is worth a missed regression + in ``config.py``. +2. **Coverage map** (``tests/.impact/coverage_map.json``) — generated from a full + run with ``--cov``. This is the only source that sees *runtime* coupling + through EventBus and Protocol indirection, which a static import graph misses + entirely. +3. **Static import closure** — the fallback for files the coverage map does not + know yet (new modules, or a stale map). Over-selects, never under-selects. + +The always-on tiers (architecture contracts, the regression ledger, the real +journeys) are never selected away; they are appended unconditionally. + +Usage:: + + python tools/impact.py --base origin/main # print the selection + python tools/impact.py --base origin/main --run # run it + python tools/impact.py --base origin/main --live-journeys + python tools/impact.py --build-map # refresh the coverage map +""" + +from __future__ import annotations + +import argparse +import ast +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_ROOT = REPO_ROOT / "src" +TESTS_ROOT = REPO_ROOT / "tests" +IMPACT_DIR = TESTS_ROOT / ".impact" +COVERAGE_MAP = IMPACT_DIR / "coverage_map.json" +ESCALATE_FILE = IMPACT_DIR / "escalate.yaml" + +# Directories whose tests never take part in selection. +ALWAYS_ON_PATHS = ("tests/regression", "tests/journeys") +ALWAYS_ON_FILES = ("tests/test_architecture_contracts.py",) + + +class FullRun(Exception): + """Raised when the change requires the whole mock layer.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +# ── Git ────────────────────────────────────────────────────────────────── + + +def changed_files(base: str) -> list[str]: + """Return repo-relative paths changed since ``base``, including uncommitted work.""" + merge_base = base + try: + merge_base = subprocess.run( + ["git", "merge-base", "HEAD", base], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() or base + except subprocess.CalledProcessError: + # No shared history (shallow clone, unfetched ref): fall back to the ref. + pass + + paths: set[str] = set() + for args in ( + ["git", "diff", "--name-only", merge_base, "--"], + ["git", "diff", "--name-only", "HEAD", "--"], + ["git", "ls-files", "--others", "--exclude-standard"], + ): + result = subprocess.run(args, cwd=REPO_ROOT, capture_output=True, text=True) + if result.returncode != 0: + raise FullRun(f"cannot determine changes ({' '.join(args)} failed); running all") + paths.update(line.strip() for line in result.stdout.splitlines() if line.strip()) + return sorted(paths) + + +# ── Escalation ─────────────────────────────────────────────────────────── + + +def escalation_patterns() -> list[str]: + """Load glob patterns that force a full run. + + Parsed with a deliberately tiny reader rather than PyYAML: this runs before + dependencies are guaranteed to be installed, and the file is a flat list. + """ + if not ESCALATE_FILE.is_file(): + return [] + patterns: list[str] = [] + for line in ESCALATE_FILE.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line.startswith("- "): + patterns.append(line[2:].strip().strip("\"'")) + return patterns + + +def check_escalation(paths: list[str], patterns: list[str] | None = None) -> None: + """Raise :class:`FullRun` when any changed path matches an escalation rule.""" + rules = escalation_patterns() if patterns is None else patterns + for path in paths: + candidate = Path(path) + for pattern in rules: + if candidate.match(pattern): + raise FullRun(f"{path} matches escalation rule {pattern!r}") + + +# ── Static import graph ────────────────────────────────────────────────── + + +def _module_name(path: Path) -> str: + relative = path.relative_to(SRC_ROOT).with_suffix("") + parts = list(relative.parts) + if parts[-1] == "__init__": + parts.pop() + return ".".join(parts) + + +def build_import_graph() -> dict[str, set[str]]: + """Return module -> set of leapflow modules that import it.""" + reverse: dict[str, set[str]] = {} + for path in SRC_ROOT.rglob("*.py"): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): + continue + importer = _module_name(path) + for node in ast.walk(tree): + targets: list[str] = [] + if isinstance(node, ast.ImportFrom) and node.module: + targets.append(node.module) + elif isinstance(node, ast.Import): + targets.extend(alias.name for alias in node.names) + for target in targets: + if target.startswith("leapflow"): + reverse.setdefault(target, set()).add(importer) + return reverse + + +def dependents_closure(modules: set[str], reverse: dict[str, set[str]]) -> set[str]: + """Return ``modules`` plus everything that transitively imports them.""" + seen = set(modules) + queue = list(modules) + while queue: + current = queue.pop() + for importer in reverse.get(current, ()): + if importer not in seen: + seen.add(importer) + queue.append(importer) + return seen + + +def test_modules_importing(modules: set[str]) -> set[str]: + """Return test files that import any module in ``modules``.""" + selected: set[str] = set() + for path in TESTS_ROOT.rglob("test_*.py"): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): + continue + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.ImportFrom) and node.module: + names.append(node.module) + elif isinstance(node, ast.Import): + names.extend(alias.name for alias in node.names) + if any( + name == module or name.startswith(f"{module}.") + for name in names + for module in modules + ): + selected.add(str(path.relative_to(REPO_ROOT))) + break + return selected + + +# ── Coverage map ───────────────────────────────────────────────────────── + + +def load_coverage_map() -> dict[str, list[str]]: + """Return the committed test -> source-files map, or {} when absent.""" + if not COVERAGE_MAP.is_file(): + return {} + try: + payload = json.loads(COVERAGE_MAP.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return dict(payload.get("tests") or {}) + + +def tests_touching(paths: set[str], coverage: dict[str, list[str]]) -> tuple[set[str], set[str]]: + """Return (selected tests, source paths the map did not know about).""" + selected: set[str] = set() + known: set[str] = set() + for test_file, sources in coverage.items(): + source_set = set(sources) + known |= source_set + if source_set & paths: + selected.add(test_file) + return selected, paths - known + + +# ── Selection ──────────────────────────────────────────────────────────── + + +def select_from_paths( + paths: list[str], + *, + coverage: dict[str, list[str]] | None = None, + patterns: list[str] | None = None, +) -> tuple[list[str], str]: + """Decide the selection for an explicit set of changed paths. + + Separated from git so the decision itself is testable: this function is what + determines whether a regression gets a chance to fail, and a defect in it + would silently shrink the suite. + """ + try: + check_escalation(paths, patterns) + except FullRun as escalate: + return [], f"full mock layer: {escalate.reason}" + + if not paths: + return [], "full mock layer: no changes detected relative to the base" + + source_paths = {p for p in paths if p.startswith("src/") and p.endswith(".py")} + touched_tests = { + p for p in paths if p.startswith("tests/") and Path(p).name.startswith("test_") + } + + if not source_paths and not touched_tests: + return [], "full mock layer: change touches neither src/ nor tests/" + + resolved_coverage = load_coverage_map() if coverage is None else coverage + from_coverage, unknown = tests_touching(source_paths, resolved_coverage) + + from_static: set[str] = set() + if unknown: + modules = set() + for path in unknown: + full = REPO_ROOT / path + if full.is_file(): + modules.add(_module_name(full)) + if modules: + impacted = dependents_closure(modules, build_import_graph()) + from_static = test_modules_importing(impacted) + + selected = sorted(from_coverage | from_static | touched_tests) + reason = ( + f"{len(selected)} test file(s) selected from {len(paths)} changed path(s): " + f"{len(from_coverage)} via coverage map, {len(from_static)} via import graph " + f"({len(unknown)} source file(s) not in the map), " + f"{len(touched_tests)} directly edited" + ) + if not selected: + return [], f"full mock layer: nothing matched ({reason})" + return selected, reason + + +def select(base: str) -> tuple[list[str], str]: + """Return (pytest targets, human-readable explanation) for changes since ``base``.""" + try: + paths = changed_files(base) + except FullRun as escalate: + return [], f"full mock layer: {escalate.reason}" + return select_from_paths(paths) + + +def always_on_targets() -> list[str]: + """Return the tiers that are never selected away.""" + targets = [path for path in ALWAYS_ON_PATHS if (REPO_ROOT / path).is_dir()] + targets += [path for path in ALWAYS_ON_FILES if (REPO_ROOT / path).is_file()] + return targets + + +# ── Journey selection (the live lane only) ──────────────────────────── + + +@dataclass(frozen=True) +class JourneyMeta: + """Declared metadata for one real end-to-end journey.""" + + path: str + subject_paths: tuple[str, ...] + live_signal: bool + + +def journey_metadata() -> list[JourneyMeta]: + """Read ``SUBJECT_PATHS`` / ``LIVE_SIGNAL`` from each journey module. + + Parsed rather than imported: importing a test module pulls in pytest fixtures + and the harness, and this runs before a test session exists. Keeping the + declaration inside the journey — instead of in a table here — means it moves + with the assertions it describes and cannot silently go stale. + """ + directory = TESTS_ROOT / "journeys" + found: list[JourneyMeta] = [] + for path in sorted(directory.glob("test_*.py")): + subjects: tuple[str, ...] = () + live = True + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if not isinstance(target, ast.Name): + continue + try: + if target.id == "SUBJECT_PATHS": + subjects = tuple(str(item) for item in ast.literal_eval(node.value)) + elif target.id == "LIVE_SIGNAL": + live = bool(ast.literal_eval(node.value)) + except ValueError: + continue + found.append( + JourneyMeta( + path=str(path.relative_to(REPO_ROOT)), + subject_paths=subjects, + live_signal=live, + ) + ) + return found + + +def select_journeys( + paths: list[str], + *, + live_only: bool = False, + patterns: list[str] | None = None, +) -> tuple[list[str], str]: + """Return the journeys a change could plausibly break. + + Used for the *live* lane, where each journey costs real tokens and real + minutes. The offline lanes never call this — they run every journey, because + replay is cheap and a suite that can be skipped will be skipped. + + A journey with no declared subjects is always selected: that is the safe + direction for a missing declaration. + """ + journeys = journey_metadata() + if live_only: + journeys = [item for item in journeys if item.live_signal] + if not journeys: + return [], "no journeys are eligible" + + try: + check_escalation(paths, patterns) + except FullRun as escalate: + return ( + [item.path for item in journeys], + f"all eligible journeys: {escalate.reason}", + ) + + source_paths = [p for p in paths if p.startswith("src/")] + touched_journeys = {p for p in paths if p.startswith("tests/journeys/")} + if not source_paths and not touched_journeys: + return ( + [item.path for item in journeys], + "all eligible journeys: change touches neither src/ nor tests/journeys/", + ) + + selected: list[str] = [] + for item in journeys: + if item.path in touched_journeys or not item.subject_paths: + selected.append(item.path) + continue + if any(p.startswith(prefix) for p in source_paths for prefix in item.subject_paths): + selected.append(item.path) + + if not selected: + return [], ( + f"no eligible journey declares a subject touched by these " + f"{len(source_paths)} source path(s)" + ) + return selected, ( + f"{len(selected)}/{len(journeys)} eligible journey(s) selected from " + f"{len(source_paths)} changed source path(s)" + ) + + +def build_map() -> int: + """Run the mock layer with coverage and write the test -> sources map.""" + IMPACT_DIR.mkdir(parents=True, exist_ok=True) + print("running the mock layer with per-test coverage; this takes a while...") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/", + "-q", + "-m", + "not e2e", + "-p", + "no:cacheprovider", + "--cov=leapflow", + "--cov-context=test", + "--cov-report=", + ], + cwd=REPO_ROOT, + ) + if result.returncode != 0: + print("the suite failed; not writing a coverage map from a red run") + return result.returncode + + import sqlite3 + + data_file = REPO_ROOT / ".coverage" + if not data_file.is_file(): + print("coverage produced no data file") + return 1 + + mapping: dict[str, set[str]] = {} + with sqlite3.connect(data_file) as connection: + rows = connection.execute( + """ + SELECT context.context, file.path + FROM line_bits + JOIN context ON context.id = line_bits.context_id + JOIN file ON file.id = line_bits.file_id + """ + ).fetchall() + for context, file_path in rows: + if not context: + continue + test_file = context.split("::", 1)[0] + if not test_file.endswith(".py"): + continue + try: + relative = str(Path(file_path).resolve().relative_to(REPO_ROOT)) + except ValueError: + continue + mapping.setdefault(test_file, set()).add(relative) + + payload = { + "_comment": ( + "Generated by tools/impact.py --build-map. Maps each test file to the " + "source files it actually executed, so change-scoped selection sees " + "runtime coupling that a static import graph cannot." + ), + "tests": {key: sorted(value) for key, value in sorted(mapping.items())}, + } + COVERAGE_MAP.write_text( + json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + print(f"wrote {COVERAGE_MAP.relative_to(REPO_ROOT)} for {len(mapping)} test files") + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Print or run the change-scoped selection.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", default="origin/main", help="Git ref to compare against") + parser.add_argument("--run", action="store_true", help="Run the selection with pytest") + parser.add_argument( + "--build-map", action="store_true", help="Refresh the coverage-derived impact map" + ) + parser.add_argument( + "--live-journeys", + action="store_true", + help="Print the live-capable journeys a change could break, one per line", + ) + parser.add_argument("--jobs", default="auto", help="pytest-xdist parallelism") + args = parser.parse_args(argv) + + if args.build_map: + return build_map() + + if args.live_journeys: + # Emitted on stdout as a bare list so a workflow can pass it straight to + # pytest; the explanation goes to stderr so it stays out of that list. + try: + paths = changed_files(args.base) + except FullRun as escalate: + paths = [] + print(f"impact: {escalate.reason}", file=sys.stderr) + selected, reason = select_journeys(paths, live_only=True) + print(f"impact: {reason}", file=sys.stderr) + for path in selected: + print(path) + return 0 + + selected, reason = select(args.base) + always_on = always_on_targets() + print(f"impact: {reason}") + + # The always-on tiers are appended, never filtered: whichever mock tests were + # selected, the real journeys and the incident ledger still run. No marker + # expression is applied, because any filter here could exclude them. + targets = (selected + always_on) if selected else ["tests/"] + + command = [sys.executable, "-m", "pytest", *targets, "-q", "-n", args.jobs] + print("impact: " + " ".join(command[2:])) + + if not args.run: + return 0 + return subprocess.run(command, cwd=REPO_ROOT).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/sync_fixtures.py b/tools/sync_fixtures.py new file mode 100644 index 0000000..189d470 --- /dev/null +++ b/tools/sync_fixtures.py @@ -0,0 +1,212 @@ +"""Derive mock-layer LLM fixtures from recorded cassettes. + +This is the join between the two test layers. The mock layer keeps its speed and +its ability to enumerate branches, but stops inventing what a provider sends: the +payload *shapes* it feeds into parsers come from real recorded traffic. When the +nightly live lane re-records and a provider has changed its response, the derived +fixtures change with it and the mock layer notices — instead of passing forever +against a body nobody has verified. + +Usage:: + + python tools/sync_fixtures.py # write fixtures, report changes + python tools/sync_fixtures.py --check # fail if fixtures are out of date + +``--check`` is what CI runs: it turns provider drift into a red build with a diff +rather than a silent divergence. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Iterable + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "src")) + +from tests._harness.cassette import CassetteStore # noqa: E402 + +CASSETTE_ROOT = REPO_ROOT / "tests" / "_fixtures" / "cassettes" +RECORDING_ROOT = REPO_ROOT / "tests" / "_fixtures" / "recordings" +FIXTURE_ROOT = REPO_ROOT / "tests" / "_fixtures" / "llm_responses" + +# Fixture files derived from the store. Each one answers a question the mock layer +# asks: "what does a successful body look like", "what does an error body look +# like", "which usage fields do providers actually send". +SHAPES_FILE = "response_shapes.json" + + +def _sse_payloads(frames: Iterable[bytes]) -> list[dict[str, Any]]: + """Parse the JSON objects carried by SSE data frames.""" + payloads: list[dict[str, Any]] = [] + for frame in frames: + for line in frame.decode("utf-8", errors="replace").splitlines(): + line = line.strip() + if not line.startswith("data:"): + continue + body = line[len("data:") :].strip() + if not body or body == "[DONE]": + continue + try: + parsed = json.loads(body) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + payloads.append(parsed) + return payloads + + +def _key_shape(value: Any) -> Any: + """Reduce a payload to its key structure, dropping volatile values. + + Only the shape matters: names of fields and their types are what parsers + depend on, while ids, token counts and text differ on every call. + """ + if isinstance(value, dict): + return {key: _key_shape(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [_key_shape(value[0])] if value else [] + return type(value).__name__ + + +def _source_directories() -> list[Path]: + """Return every per-journey store to distil shapes from. + + Both stores contribute: ``recordings/`` is real provider traffic and is the + authority on wire shape, while ``cassettes/`` covers the injected failure + bodies (429, context overflow) that a live provider will not produce on + demand. Together they describe every shape the parsers must handle. + """ + directories: list[Path] = [] + for root in (RECORDING_ROOT, CASSETTE_ROOT): + if root.is_dir(): + directories.extend(sorted(p for p in root.iterdir() if p.is_dir())) + return directories + + +def collect_shapes() -> dict[str, Any]: + """Walk every stored exchange and summarize the shapes it contains.""" + successes: list[Any] = [] + errors: list[Any] = [] + chunks: list[Any] = [] + usage_fields: set[str] = set() + finish_reasons: set[str] = set() + error_codes: set[str] = set() + total = 0 + + for directory in _source_directories(): + store = CassetteStore(directory) + for key in store.keys(): + record = store.get(key) + if record is None: + continue + for response in record.responses: + total += 1 + if response.is_stream: + for payload in _sse_payloads(response.frames): + chunks.append(_key_shape(payload)) + usage_fields.update((payload.get("usage") or {}).keys()) + for choice in payload.get("choices") or []: + if choice.get("finish_reason"): + finish_reasons.add(str(choice["finish_reason"])) + continue + if not response.body: + continue + try: + payload = json.loads(response.body.decode("utf-8", errors="replace")) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + if response.status >= 400 or "error" in payload: + errors.append(_key_shape(payload)) + code = (payload.get("error") or {}).get("code") + if code: + error_codes.add(str(code)) + continue + successes.append(_key_shape(payload)) + usage_fields.update((payload.get("usage") or {}).keys()) + for choice in payload.get("choices") or []: + if choice.get("finish_reason"): + finish_reasons.add(str(choice["finish_reason"])) + + return { + "_comment": ( + "Generated by tools/sync_fixtures.py from tests/_fixtures/recordings " + "(real provider traffic) and tests/_fixtures/cassettes (deterministic " + "replay inputs, including injected failures). Do not edit by hand: run " + "`make sync-fixtures` after re-recording." + ), + "stored_responses_seen": total, + "completion_shapes": _dedupe(successes), + "chunk_shapes": _dedupe(chunks), + "error_shapes": _dedupe(errors), + "usage_fields": sorted(usage_fields), + "finish_reasons": sorted(finish_reasons), + "error_codes": sorted(error_codes), + } + + +def _dedupe(shapes: list[Any]) -> list[Any]: + """Return unique shapes in a stable order.""" + seen: dict[str, Any] = {} + for shape in shapes: + seen.setdefault(json.dumps(shape, sort_keys=True), shape) + return [seen[key] for key in sorted(seen)] + + +def main(argv: list[str] | None = None) -> int: + """Write or verify the derived fixtures.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--check", + action="store_true", + help="Fail when the committed fixtures differ from the cassettes", + ) + args = parser.parse_args(argv) + + if not CASSETTE_ROOT.is_dir() and not RECORDING_ROOT.is_dir(): + print( + f"no stored exchanges at {CASSETTE_ROOT} or {RECORDING_ROOT}; " + "run `make seed-cassettes` first" + ) + return 1 + + shapes = collect_shapes() + rendered = json.dumps(shapes, indent=2, ensure_ascii=False) + "\n" + target = FIXTURE_ROOT / SHAPES_FILE + + if args.check: + if not target.is_file(): + print(f"missing derived fixture {target}; run `make sync-fixtures`") + return 1 + current = target.read_text(encoding="utf-8") + if current != rendered: + print( + f"{target.relative_to(REPO_ROOT)} is out of date with the stored " + "exchanges — a provider response shape changed.\n" + "Run `make sync-fixtures`, review the diff, and commit it." + ) + return 1 + print(f"{target.relative_to(REPO_ROOT)} is up to date") + return 0 + + FIXTURE_ROOT.mkdir(parents=True, exist_ok=True) + changed = not target.is_file() or target.read_text(encoding="utf-8") != rendered + target.write_text(rendered, encoding="utf-8") + verb = "updated" if changed else "unchanged" + print( + f"{verb}: {target.relative_to(REPO_ROOT)} " + f"({shapes['stored_responses_seen']} stored responses, " + f"{len(shapes['completion_shapes'])} completion shapes, " + f"{len(shapes['error_shapes'])} error shapes)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index 48cf6c5..57e7837 100644 --- a/uv.lock +++ b/uv.lock @@ -376,6 +376,95 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/38/ce65091ff20a16e06d17418c4353af5f56d3190821b1a06983c79ae79274/courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e", size = 34193 }, ] +[[package]] +name = "coverage" +version = "7.15.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/45/78dbf9604ee5b3db24efbf26bed1cb58862fb40480cba821963c69348751/coverage-7.15.3.tar.gz", hash = "sha256:ae7ea5a4614acf399ef0483c4cb34f8f8f01df848d8fcbe7d3ce0865733f1c4d", size = 935592 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/9c/c8a3a923c24f631695cea2d5e2f02e776bc0af6e03800626e13a6c05a615/coverage-7.15.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5f3f854ab4599d98f7799ac9b91e34e8ec9ebc9a6372ee8c1f3413a68cc8b5e9", size = 222328 }, + { url = "https://files.pythonhosted.org/packages/92/51/dda77f34cbd2513d6ffb898c901d19e9ca55f48c0cbc4a1eb173a97d157a/coverage-7.15.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75268348fee1f199653b8a846262aec5581c6bb008c4f58824959fb708cc688f", size = 222832 }, + { url = "https://files.pythonhosted.org/packages/78/59/e0faafc4c6e23bd76c76148875ee9ec5781b8f1cd62cea2bc4ca0f0f0e5d/coverage-7.15.3-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:21081739f6264cc594cad2d42b62befbd17633824022866c68720eb0c4b8d6b4", size = 253250 }, + { url = "https://files.pythonhosted.org/packages/14/e2/4b1e0eeb727ffb471e411c1bd3402184b5dd54a77a762b0e55e87cdf9ae3/coverage-7.15.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:718d366251b060c10731c7dd359de6caea72250036eb94576aa56dacbf830a11", size = 255160 }, + { url = "https://files.pythonhosted.org/packages/e9/9e/a602d2d48f9db9f795e578a86aa914f7b20008e9330902defcfb73d17b3a/coverage-7.15.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fa1bbaa502a6e877f3ee67cbac3eba2bb637f623e454e6c37b81b38896dbd48f", size = 257269 }, + { url = "https://files.pythonhosted.org/packages/22/fa/bf6db13df2fcee00d2671849fe58c99232ee79a01fec7478c2bf7839b9e1/coverage-7.15.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:494880c9e60782610683f4eb9b65cce4f886673596b8f3cb2dfa079fc551c743", size = 259231 }, + { url = "https://files.pythonhosted.org/packages/89/37/8118f13b17fa7d9a3aa2c301d93f2d5ffeef70fa7e27e639a74bdacd3fea/coverage-7.15.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3db264ea689f9e8f9fa4fb9005fee4048c3bff4a547f4cfa27f5086cb0804ec0", size = 253357 }, + { url = "https://files.pythonhosted.org/packages/97/6d/c7b94fb03962f4d6f0fe13d01c4eb9c4c6e2e714a20d074516ec7582b110/coverage-7.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4e869d4799674d67778e76ddbe2e26cf1673369262e231a8ec259421b1015fea", size = 254961 }, + { url = "https://files.pythonhosted.org/packages/87/f9/fe0bd415fa56e36b62b649017c8fc98330858be4c7593789efb78cd24178/coverage-7.15.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:696fc7a28bbf717aba8d2c6963d26702945c7832cb313ba3b323aa5b1afb3156", size = 253024 }, + { url = "https://files.pythonhosted.org/packages/c1/7c/ffa53506d63ba8a77f5b9557dd6f5a5a5ad85adc680d7857410138f82bd9/coverage-7.15.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3fe9be1c527497d047f770d88a0110189714c36383bb88384508f750c302bffa", size = 256792 }, + { url = "https://files.pythonhosted.org/packages/1f/c6/df42458e72c18a49fe87e40ccd3fb0314210915256cf4a5593e1b3250e04/coverage-7.15.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2400591f4b2e33746c70846388f8bb4c7e33b820e31cb8c6cb2f25305310438b", size = 252744 }, + { url = "https://files.pythonhosted.org/packages/f1/14/8bf18a4b10a44f8ba5f604b00e102f37daf49d581d66a37dc33fa267e1a6/coverage-7.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2e557178799282269412a672e5753f2179edfe1b3f0f19b0c98f8e72d482326a", size = 253652 }, + { url = "https://files.pythonhosted.org/packages/27/e6/e530c9bb94e4155817cbd149034105b062a6913bc356ae08f454d155de53/coverage-7.15.3-cp311-cp311-win32.whl", hash = "sha256:68ea6c947375982ae907e19e9d2ef156bd6e68e11f3566dd568d7f4ec974e715", size = 224428 }, + { url = "https://files.pythonhosted.org/packages/b4/98/0050c692d120988f1973a15196f52dee4ae221848b760281461a2005b613/coverage-7.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:28743dad31622e8c474b17446118037361f5b1f4f2ecdf72d4f6fde246d64446", size = 224906 }, + { url = "https://files.pythonhosted.org/packages/b0/ae/c0ef3e2ba3f35fc1c6985811a40edd9331e5b8978c9ecf84699de3edacbe/coverage-7.15.3-cp311-cp311-win_arm64.whl", hash = "sha256:c4398918c4fda32718191239e451fd86ac5ad1e8979b592f1921ee2d1f038965", size = 224448 }, + { url = "https://files.pythonhosted.org/packages/d1/6c/bac99d9d4c6abe856e93bf3f5212982ac0bfac126dd4a042753bd53bc5af/coverage-7.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:79a3e32e83227d83d9684459ed579769b56c369ac2d7313099b2d9e031d2e10f", size = 222499 }, + { url = "https://files.pythonhosted.org/packages/aa/bc/cb9a39b083bc1aa70586482dab25c9be20bab0ec6c155340e50d9066bb1e/coverage-7.15.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:767feb87c5886d781d0a69fafd450a20826ddab7b79bce1665deb64d21441b60", size = 222866 }, + { url = "https://files.pythonhosted.org/packages/58/fb/beaa453d62000a0a5b39838bee2a137afe609a50a71f55e83c73461e513b/coverage-7.15.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:50951e37033c40548d777b8a8454a2cd622dba1136780065678dccaec307c47f", size = 254367 }, + { url = "https://files.pythonhosted.org/packages/66/64/43e72500ed6815cef189f9193f29d7af4b078830337c95ea976cd0c0d427/coverage-7.15.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:63a4ff67364afb2cac826b8bbd78a5c50ce656a7b7137436b44d7b96a9271088", size = 257103 }, + { url = "https://files.pythonhosted.org/packages/66/3a/2893e2937adfe02f45fd38e4a8a0a0d8b7a02ff9e012ac3d009bee3c4f16/coverage-7.15.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e95e42856509675fe26560310313a6117640e96f9a1e19bb3d220116a27c94c", size = 258220 }, + { url = "https://files.pythonhosted.org/packages/30/b4/d5e6e2eb1a62961083734291304b1f85df72e2abe95c76eb88a7f472afd0/coverage-7.15.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:abad631cba27094b4631993f4c72e89ac0ca1b3a0236c7abaf8ca79aea619851", size = 260481 }, + { url = "https://files.pythonhosted.org/packages/dc/c9/9b72c5c6a9798a9a12cf65f66e077cc1fdd396e61915c862688f9afe1cae/coverage-7.15.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2b0807f1f051dd82a234ad6acdb6f1425baede60be1e84e862496c8cc9262ab9", size = 254749 }, + { url = "https://files.pythonhosted.org/packages/92/20/e1c2f759e2dbce559ba85c40c0e4acfecc6cff4b740c294c88e41ccc6111/coverage-7.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8d6df7aeb5bc464040bbc9ae173d875785d3677ebc4307817997d622d74225e", size = 256138 }, + { url = "https://files.pythonhosted.org/packages/a5/ab/48cc7e760f769e86ae290a125ea6e7209dfbdbbbb7ff4f5d9d1ee7a45d57/coverage-7.15.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:974471c506c9f5758808b47c1ebf7949ecd0848f5c1020e78675fefe5ff46866", size = 254283 }, + { url = "https://files.pythonhosted.org/packages/15/26/39529a68154f99b3a1829debd8b25eac384effeec890a293b5bbdcb49186/coverage-7.15.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5cba0c9c13e35c86df7998f1afaf6b1da224a3a39e4da59bdabf60c148046dcb", size = 258352 }, + { url = "https://files.pythonhosted.org/packages/91/2f/55b82aa3d8d7dd8023a56e7c5c2a70e39a3c44b3353c6cf3faec9ad51566/coverage-7.15.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4d608dc36a364dce33acbf4fc3a50f9d2054c945f233bb0a2cdb4b90bfa17646", size = 253852 }, + { url = "https://files.pythonhosted.org/packages/6a/6d/839f4045124cd3518ecf2c58967e58a911202834e7c5a03cfdf2ab0b29f6/coverage-7.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2395869280554a1941da904423c12660c39f721315e1c02d076a7fe0971382f0", size = 255725 }, + { url = "https://files.pythonhosted.org/packages/75/21/d25e3e2a9e327798078c877f469dfb6def860bf6e25036529046227d3e15/coverage-7.15.3-cp312-cp312-win32.whl", hash = "sha256:24f3b21840c3eb76cef3cc70b2bf6649010c64471a84a446538a39306e1ba04d", size = 224566 }, + { url = "https://files.pythonhosted.org/packages/b1/0f/df90cc1e8d095ce263968a93e04829821b2afb31ac2752c06a2e0a8e3c13/coverage-7.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:fa7b17902c3c1dd8a7adb52679b7f6340bba08443d710c8838e04db8cf62be2a", size = 225098 }, + { url = "https://files.pythonhosted.org/packages/65/c7/ec49e43c58967a07163e2d1c6bbd58112b825b2772ab66784afd6a5400ba/coverage-7.15.3-cp312-cp312-win_arm64.whl", hash = "sha256:fcbe83fb7258eacd293bf5322d88807acb35ed12a5cfa99dd8215c083e3b0235", size = 224485 }, + { url = "https://files.pythonhosted.org/packages/68/6e/62ae61e1fc434956bec38ed1d5b1c494f58cf579dbd998e77abffe7b3e6b/coverage-7.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1182eed05674c63d40951fae27c43e822749f04d25f75df64c2e4fa3168678de", size = 222522 }, + { url = "https://files.pythonhosted.org/packages/13/ff/c74c673d81e0e77b6608c3d21331e3db42e30daeb3c8a0a8860d4c9e2e14/coverage-7.15.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c0c4b0d7c4cd56e470d0c9d8441f42e8a96cdfd95050fec027f1d4dd9f11006c", size = 222894 }, + { url = "https://files.pythonhosted.org/packages/a1/91/ccb30f5ffafd7d69d0b18e5162f9b711a5654e807b7b0c13497f0826b33f/coverage-7.15.3-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5c9fce9f4998b0d50a753da765b9215a14decc7863822c89d72da7a89ca625b3", size = 253890 }, + { url = "https://files.pythonhosted.org/packages/29/c6/e92a66cda49a2751b09826d51258f199b92aa0cb005bc5f34e9729a52a9c/coverage-7.15.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7a47e2a0a0ace9241e70ee00e44520f88b843094603dd54303f1bafecd929c30", size = 256484 }, + { url = "https://files.pythonhosted.org/packages/96/7a/730929164b457cf25cf76c23898b90f9039a104a647890801b6586797b14/coverage-7.15.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95bad94f83807ae60ed76f3ac012f69b2605ac9ea81bee959a5a483f7fa09c10", size = 257723 }, + { url = "https://files.pythonhosted.org/packages/9e/be/04cb5672cb19f5c389eda81ba22d89807699a949653d3625b0e0fda169da/coverage-7.15.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:228e172a76c428bb17d1ab78a2ff188990b0597e5dbd291f52a4edf7412de049", size = 259854 }, + { url = "https://files.pythonhosted.org/packages/96/25/5e7fd6af39f6507071455944b8906dd1fe5b7b6bffb6a163ceb20afa0d13/coverage-7.15.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cea9fb33887c99349996266f1fd60abe5af3577a90633392001d27ef46b4b66e", size = 254085 }, + { url = "https://files.pythonhosted.org/packages/23/c8/55e58a853f1e61163a6e755897bd14a059d78411e86560f39d9951c019b5/coverage-7.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81760de3155d7f52c21860c4046628dc6bed182f72e3c028e2b4fd46f65aa040", size = 255850 }, + { url = "https://files.pythonhosted.org/packages/be/74/8bcec66dbcf3d22bea2a0b2b77ee2fa6f766a647d0023d4eabbc4f2b2756/coverage-7.15.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b47ea0a1d3a3d089826c6cbfad8429d7d8872e28e86baa95ddef330f6875da21", size = 253818 }, + { url = "https://files.pythonhosted.org/packages/ce/06/450b673fdfece0997b4e16a31d6bde6b18889c578f1013ddd34c962ac6f9/coverage-7.15.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5459ba486b2a5d58a6c05254779ecdf525e7f20174d0210ceda75ba40fdb8f2c", size = 257973 }, + { url = "https://files.pythonhosted.org/packages/56/fd/3ec7409aec0ddc943132452b65672f065f043b844f1830e1fe173c98b3ab/coverage-7.15.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c59209f80a08dbfcdd5109a80dc623cd3b9d22895c85757d34f57a6e6e95570f", size = 253638 }, + { url = "https://files.pythonhosted.org/packages/75/20/30a8dabb194123631c93f860fdd86401ad405d56cfb1841873afbfe4e92b/coverage-7.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f863856c1779d4a5bb6a94698a2f9073e09c6706501f76f3e7780e72df97d21c", size = 255407 }, + { url = "https://files.pythonhosted.org/packages/13/4d/e14365b1953b43653341412f9088b0d752614c626a73a705ff9af400f3a3/coverage-7.15.3-cp313-cp313-win32.whl", hash = "sha256:00cbdc5e322927dc30c5e42b863819b1bb867cc66f26ab5372c585850876ab93", size = 224575 }, + { url = "https://files.pythonhosted.org/packages/1c/64/88f762ea80de2070207246faef514513be874486b2773528f2cc2b4b515c/coverage-7.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:835528518a1d823cf336740324b2f335f7c01e609e74abcb5d5163b3e66661e3", size = 225116 }, + { url = "https://files.pythonhosted.org/packages/ab/66/03c34c53a319f522554cd29d4f2e16c5eab61aa4cdcf55753129fd7d926c/coverage-7.15.3-cp313-cp313-win_arm64.whl", hash = "sha256:0d2e1f2cbbf36b842f3e2aff8d118c60d677adb498bc6c7fa9c6838738f82767", size = 224509 }, + { url = "https://files.pythonhosted.org/packages/35/6f/8c2dc014357618b3226c90f731b8282766c3685786f422558991dc49fbf2/coverage-7.15.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1e3bb08ad574bd9fb6a991f645728f70d333c1c1958dd5fcde65e24cb862813d", size = 222571 }, + { url = "https://files.pythonhosted.org/packages/07/50/d867c7ceae9d56b7e74ee61ea834f1aa4f9a1e1c7f0ce39393ba573b1c12/coverage-7.15.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e5860eaff02a0b7f1b73304bdf846596ee62ab3a78d25c68044ebf684cb1fef", size = 222902 }, + { url = "https://files.pythonhosted.org/packages/62/77/4f6dfc490c5f2bcacb2d296d9aa4d1e128c43b48e94ad313fec7f49f09ad/coverage-7.15.3-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:60874e5bd67f0b1bdbe42ab42c7bafa66a6fb8de88721af6df3f7a02713960cd", size = 253947 }, + { url = "https://files.pythonhosted.org/packages/16/8a/6777f192af264165103e2a3d3768dbadb9894a0a2359a16877141d9ae8f5/coverage-7.15.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9147be876e9d83765e0b82176674dc248a6b9283e25e01e7462611b97e9b731", size = 256452 }, + { url = "https://files.pythonhosted.org/packages/7d/7b/3d7ac46a0234bc684f41ee42be95e29b2b6525695adb04083609d5ac2149/coverage-7.15.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61a01f8c3804760fcc5a3d31c4f3cab792d660d44e17bf7adeaf0ea51e07821e", size = 257798 }, + { url = "https://files.pythonhosted.org/packages/ff/1e/c6ee59c29afcb5fdb35f936381340d1a06429a07c48f20e809646647acbe/coverage-7.15.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95bf3e7f26f792e25eb185f85a5a659d48479265176dcfe22b6f334fd0081b5c", size = 260112 }, + { url = "https://files.pythonhosted.org/packages/c1/e1/e8ea39a46e89e3a143312ee5f80336e992e3ae8fe44bf9c76b83fefeed42/coverage-7.15.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:44c41eff9e413fed8740eca75d5438ebeb9d3e45e7cd37c67329213e7a72c764", size = 253944 }, + { url = "https://files.pythonhosted.org/packages/95/67/31ab5f6a37fd887d1386f81f0da9306851ad2264e9baaa9c7f606e0b3e17/coverage-7.15.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54146bafb61f3ba9895b43af0dd17eba01561d586d44ce84ea221b0cbbee5a9e", size = 255805 }, + { url = "https://files.pythonhosted.org/packages/fb/6a/ee505a80c8fd89620fb337c0596daecff87f33171fbb4ee3015fc3d7331f/coverage-7.15.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:af000dd1bb859ff8066fda4c79512ff938c798116540307226b373099c7b151f", size = 253769 }, + { url = "https://files.pythonhosted.org/packages/b0/41/6ab0f81c9e89660230d8f3f581d4732e5ddb75a885b0a5dfc73d315dc94f/coverage-7.15.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a1b82490577f3889950b5a04f18712aef0207243e0749d60fe28c3c73ebfd5fd", size = 258045 }, + { url = "https://files.pythonhosted.org/packages/bc/62/c995e91cae28cf31d6defab3bfb553dda5ac83ac7381b0f2b121264c307a/coverage-7.15.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:c4fc90a60154c3e4b8a2dc206d6dbe852f1c235c249e0dc0cef909d032c9591a", size = 253587 }, + { url = "https://files.pythonhosted.org/packages/84/df/f2049980f82d6890321f2065f9e66216eabbf4b2001815db958bc543f40a/coverage-7.15.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f25bb884814a892948b4c20394db3f2364dd452d9492736479e7a493e63b0eb6", size = 255243 }, + { url = "https://files.pythonhosted.org/packages/1d/82/2c841b67a978c0eb9c3707630b68f93f9e7585d78bb906bc8823ec6b07a5/coverage-7.15.3-cp314-cp314-win32.whl", hash = "sha256:722dbf8e7828fbcfe0dc8586167dc0a5ce85ad6ea171dbb21ed3f8d6581d3cb8", size = 224759 }, + { url = "https://files.pythonhosted.org/packages/b3/78/5c93ec43784fd3e404ca23cd0584ae24bc1732de4a3fc194b68c3be88db0/coverage-7.15.3-cp314-cp314-win_amd64.whl", hash = "sha256:64d0845f9c3ed47302bed265c15ab4dbb64aa4ec1490839b8e328f4e7fa914d2", size = 225246 }, + { url = "https://files.pythonhosted.org/packages/9d/77/813a054371f3b018cc63c6bdb46a3c35d5e95d4e3ed4f1449d4196106db5/coverage-7.15.3-cp314-cp314-win_arm64.whl", hash = "sha256:69bc14684f8fbbee9f9dbaa4fe79719b0da9725fc37956785c06ec365acf6926", size = 224673 }, + { url = "https://files.pythonhosted.org/packages/8f/63/8c9f36cc71178d26db930baa03a4494abcc516d8d41bf820d0d85ef1d80b/coverage-7.15.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f92df943c24b96cb215ca26b4f6a2283e63c5db80f1635aceea7fff11311917b", size = 223298 }, + { url = "https://files.pythonhosted.org/packages/54/66/211f24d058ce9f56ebf1420d55b7574fdae924f6da3836f83c8bd4793e38/coverage-7.15.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:66591c46bdd2971d3ae2bc503a5f0459c2edcaf6b7e045b292000cc95bc6cb95", size = 223568 }, + { url = "https://files.pythonhosted.org/packages/dd/bb/9c2ad5574a0d6420a96c6cade4f8a683931b9e79fe609f8924d7b6964616/coverage-7.15.3-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa64458b81b18bfc67cdf1f6dc02b23e3edc672f2f8e11771fad75865415a43", size = 264932 }, + { url = "https://files.pythonhosted.org/packages/ba/91/938c39e77bdd5a0a440412f975609ce3702dabbda6ac715719d93ca45a7b/coverage-7.15.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:447f5421ccf5475956cf516d4ca1d575f487947b6f4e11f9d80c6aefe24b3dc8", size = 267052 }, + { url = "https://files.pythonhosted.org/packages/b0/a3/7b431a98af35d9cc6394e54cde9435b33b8591672fbece6a4931267d7a8e/coverage-7.15.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a0c77ef8cd483a4987a5d12d1d9d5f7ee598dfdc6c0844417d847e5768dc779", size = 269473 }, + { url = "https://files.pythonhosted.org/packages/32/58/dbc9951dce46be47a732823a1c571f62bcabdd54a68d8c281489a1a55cfb/coverage-7.15.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b273f4ff657446a06c2d85bf80e134fa869a92852ba5f87854a70e1fb44da77", size = 270591 }, + { url = "https://files.pythonhosted.org/packages/71/bd/1d610772c7c0889bfe477a59c46ee66ea53e271f3f06951e9d55b317f7c6/coverage-7.15.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daea8c4fafa22488600405be2c2be525a9406fba3fc0a83acc726db3e14e2005", size = 264007 }, + { url = "https://files.pythonhosted.org/packages/69/97/852eb3dcdba156b1a9078503f098499916bf889f964b61ad4a08223ac169/coverage-7.15.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:93ff57c530f3fa7aa69f92fb9b8892b8aa82712aa970842f4abf28657f42fb57", size = 266926 }, + { url = "https://files.pythonhosted.org/packages/52/f8/b72cd238757fba2b587fc7dee047efe6e10b0c18343509faaaf502dd4680/coverage-7.15.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4df21bef8b800eebda9018f53d49c9ace3aeb0090c850139b27923aafcb83e91", size = 264529 }, + { url = "https://files.pythonhosted.org/packages/b8/0a/6c52ec4b7fb007cb6433d1fcfda4080cb15d75ad37ef9c31025f3427293e/coverage-7.15.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:db567b02685f26034adcbd85055f80d12cdf02111b8ed00886093d98b2874ce2", size = 268263 }, + { url = "https://files.pythonhosted.org/packages/c0/4e/f1f9aa3efd109a04353563a43fb5155340c1fdcdeaa6296ebed3b6f510ea/coverage-7.15.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5318dd51b8600b947e058cf5a4fe54d183d9d13c49b97b64ca7be05a34df9bef", size = 263377 }, + { url = "https://files.pythonhosted.org/packages/dd/fb/6b268a0b2728ef1c379ad656b899274477a5f6bed1bf6765b4b387fb0601/coverage-7.15.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c995bfa383c54704839b6c4c2627a1c00895597ada0e5e8190c81d8bd620555c", size = 265688 }, + { url = "https://files.pythonhosted.org/packages/29/54/1a3ea96e5d5e7cd41dc432597bfc60692910e635d05e1cc25a8ccc243581/coverage-7.15.3-cp314-cp314t-win32.whl", hash = "sha256:6433fafb8da0e1d02eb53411e0ecdadb6b88f0224fdc23317e703c0e88937d42", size = 225066 }, + { url = "https://files.pythonhosted.org/packages/31/9d/a7b0d9afd18ed5274dd00651a78e7810a931c70d94b79996f150bec1a30f/coverage-7.15.3-cp314-cp314t-win_amd64.whl", hash = "sha256:fe578952b1b29fe8c777f43f241d49efac4b56724a3434f5d22ebe3c208df429", size = 225897 }, + { url = "https://files.pythonhosted.org/packages/ca/11/34c5ae40b945e69aa72b87dc268135b7049905f3824af573b7073acbb946/coverage-7.15.3-cp314-cp314t-win_arm64.whl", hash = "sha256:d2e1acb7aee29dfa8f3e48c23f36670898baca1209d9bdd3985a50c7f982165e", size = 225212 }, + { url = "https://files.pythonhosted.org/packages/37/e7/7069b3d6c018917f49ba2e1c5fb910e498c7fefa3a1b78cb1b79e61ff45d/coverage-7.15.3-py3-none-any.whl", hash = "sha256:da78fa6fc7dafe4212839173133ee85afcf42c5cd5f3e47fa7c1c210453b445e", size = 214297 }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + [[package]] name = "cryptography" version = "49.0.0" @@ -492,6 +581,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/e3/9d34173ec068631faea3ea6e73050700729363e7e33306a9a3218e5cdc61/duckdb-1.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:c9f3e0b71b8a50fccfb42794899285d9d318ce2503782b9dd54868e5ecd0ad31", size = 14402513 }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708 }, +] + [[package]] name = "filelock" version = "3.29.4" @@ -860,6 +958,8 @@ dashboard = [ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, ] hub = [ @@ -884,6 +984,8 @@ requires-dist = [ { name = "prompt-toolkit", specifier = ">=3.0.40" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "rich", specifier = ">=13.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4.0" }, @@ -1681,6 +1783,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075 }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876 }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396 }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2144,6 +2273,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743 }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704 }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454 }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561 }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824 }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227 }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859 }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204 }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084 }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285 }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924 }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018 }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948 }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341 }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159 }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290 }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141 }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847 }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088 }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866 }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887 }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704 }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628 }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180 }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674 }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976 }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755 }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265 }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726 }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859 }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713 }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084 }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973 }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223 }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973 }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082 }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490 }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263 }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736 }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717 }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461 }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855 }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144 }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683 }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196 }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393 }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583 }, +] + [[package]] name = "tqdm" version = "4.67.3"