From 1c94327eb4545d04a7ccc6d14916117910b9dc05 Mon Sep 17 00:00:00 2001 From: OpenClaw Date: Fri, 17 Jul 2026 00:07:23 -0500 Subject: [PATCH 1/2] fix(cli): synthesis skip + zero-cost recording on single-model ensemble path, help/output polish - panel run --models (weight-free, single entry) was classified as an ensemble and took the ensemble branch: no synthesis, no cost summary, exit 0 with a bare "Ensemble complete". Single-entry specs now demote to the standard single-model path (synthesis included) with a stderr note; ensembles require >= 2 models. - resolve_cost trusted a provider-reported cost of exactly $0 verbatim (OpenRouter returns usage.cost: 0 for BYOK keys), recording token-consuming runs as $0.0000 and starving CostGate so --max-cost could never trip. A zero provider cost with nonzero tokens now falls back to the local pricing-table estimate; genuinely free models (local table also $0) still record $0. - Ensemble text stdout now states models/personas/questions, incident count, recorded cost, "Synthesis: skipped" plus the exact `panel synthesize ` follow-up, and where results live; JSON carries synthesis_status: "skipped" explicitly. Standard text runs end with a RUN SUMMARY block (size, errors, cost, synthesis status, results path or --save hint). - --personas/--instrument help now documents bundled pack/instrument names (pack list / instruments list) alongside YAML paths. - New `cost show `: per-run cost breakdown as a thin alias over panel inspect's data; saved-result hint updated. - Regression tests in tests/test_r4_cli_fixes.py. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 44 +++++ src/synth_panel/cli/commands.py | 185 +++++++++++++++++- src/synth_panel/cli/parser.py | 29 ++- src/synth_panel/cost.py | 19 ++ src/synth_panel/main.py | 4 + tests/test_r4_cli_fixes.py | 331 ++++++++++++++++++++++++++++++++ 6 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 tests/test_r4_cli_fixes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 72147b8..59199dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ For auto-generated release notes, see [GitHub Releases](https://github.com/DataV ### Added +- **`cost show `.** Per-run cost breakdown for one saved + panel result — total/panelist cost, per-model token + USD rollup, and + synthesis cost — as a thin alias over the cost section of `panel + inspect` (same result resolution: ID or JSON path). Previously the + `cost` subcommand only had `summary` (cross-run aggregate) and per-run + cost required reading the full `panel inspect` output. The + "Result saved" follow-up hint now lists it. + - **Synthesis failure recovery ladder (sp-rcvr).** A thrice-reproduced production failure — a 20-persona panel whose questions embed ~14k-char page dumps passed the synthesis pre-flight yet deterministically failed @@ -41,6 +49,42 @@ For auto-generated release notes, see [GitHub Releases](https://github.com/DataV ### Fixed +- **A one-entry `--models` spec no longer routes through the ensemble + path, silently skipping synthesis.** `panel run --models ` + (weight-free spec, single entry) was classified as an "ensemble" and + took the ensemble branch, which never runs synthesis, never engages the + standard-path cost summary, and exited 0 with a bare "Ensemble + complete" on stdout — a naive single-model run got no synthesis and no + signal that anything was skipped. Single-entry specs are now demoted to + the standard single-model path (identical to `--model X`, synthesis + included) with a stderr note; ensemble comparison requires two or more + models. +- **Runs with token usage no longer record $0.0000 when the provider + reports a zero cost.** OpenRouter returns `usage.cost: 0` for BYOK keys + (and some upstreams omit native cost); `resolve_cost` trusted the zero + verbatim, so a run with tens of thousands of tokens recorded + `total_cost $0.0000` / `cost_usd 0.0` — and, because the orchestrator's + cost gate accrues via the same path, `--max-cost` could never trip. + A provider-reported $0 for usage the local pricing table prices as + nonzero now falls back to the local estimate (with a stderr-visible + warning); genuinely free models still record $0. Dry-run estimates and + synthesis costs (which always used the local table) were already + correct — the mismatch between them and the $0 run cost was the tell. +- **Ensemble runs now state synthesis status and results location on + stdout.** Text-mode output for a completed (>=2 model) ensemble run was + the single line "Ensemble complete"; it now prints a terse summary — + models x personas x questions, incident count, recorded cost, + "Synthesis: skipped" with the exact `panel synthesize ` + follow-up (or the `--save` prerequisite), and where the results live. + JSON output carries `synthesis: null` + `synthesis_status: "skipped"` + explicitly. Single-model text runs likewise end with a RUN SUMMARY + block (personas, questions, errors, recorded cost, synthesis status, + results path or the `--save` hint). +- **`--personas` / `--instrument` help text documents bundled names.** + Both flags said "Path to a YAML file..." although bundled + pack/instrument names (e.g. `general-consumer`, `general-survey`) are + accepted and are the primary happy path; the help now points at + `pack list` / `instruments list` and keeps the YAML-path alternative. - **Synthesis judge final-strike escalation now stays in the run's provider family for every provider (GH#571, sy-549 class).** A native `--model gemini` run with only `GEMINI_API_KEY` set could complete all diff --git a/src/synth_panel/cli/commands.py b/src/synth_panel/cli/commands.py index 5b785c0..f137f6e 100644 --- a/src/synth_panel/cli/commands.py +++ b/src/synth_panel/cli/commands.py @@ -1690,6 +1690,26 @@ def system_prompt_fn(persona: dict) -> str: except ValueError as exc: print(f"Error parsing --models: {exc}", file=sys.stderr) return 1 + # gh-r4: a one-entry weight-free spec ("--models X") is a plain + # single-model run, not an ensemble. Routing it through the + # ensemble branch silently skipped synthesis, exited 0 with a bare + # "Ensemble complete", and bypassed the standard-path cost gate / + # summary. Demote it to the same path "--model X" takes; ensemble + # comparison needs >= 2 models. (--blend keeps its own validation.) + if ensemble_mode and not blend_mode and len(model_spec) == 1: + model = model_spec[0][0] + args.model = model + has_model = model + has_models = None + model_spec = None + ensemble_mode = False + print( + f"Note: --models with a single model ({model}) runs a standard " + "panel run (synthesis included). Ensemble comparison requires " + "two or more models.", + file=sys.stderr, + ) + if has_models: # sp-zdul: warn if weighted sum is far from 1.0 — the user likely # intended an exact-ratio split and a typo (e.g. "0.3,0.3,0.3") # silently reweights their panel. @@ -1968,6 +1988,12 @@ def system_prompt_fn(persona: dict) -> str: from synth_panel._runners import format_panelist_result as _fmt_panelist output = build_ensemble_output(ens_result, panelist_formatter=_fmt_panelist) + # gh-r4: the ensemble path never runs synthesis. Say so explicitly + # in the payload instead of leaving consumers to infer it from a + # missing key ("panel inspect" showed only 'Synthesis: not run' + # with no explanation, and text stdout said nothing at all). + output["synthesis"] = None + output["synthesis_status"] = "skipped" # sp-atvc: attach a metadata bundle whose cost.per_model covers # every ensemble model, not just the first in the spec. ens_per_model_meta = {mr.model: (mr.usage, mr.cost) for mr in ens_result.model_results} @@ -2021,7 +2047,41 @@ def system_prompt_fn(persona: dict) -> str: output["result_id"] = ensemble_result_id output["saved_path"] = str(_results_dir() / f"{ensemble_result_id}.json") - emit(fmt, message="Ensemble complete", extra=output) + # gh-r4: text-mode stdout used to be the single line "Ensemble + # complete" — no cost, no error count, no synthesis status, no + # pointer to the results. Print a terse run summary instead. + # JSON/NDJSON keep the same message and carry everything in extra. + if fmt is OutputFormat.TEXT: + n_models = len(ensemble_models) + incidents = output.get("ensemble_incidents") or [] + print( + f"Ensemble complete: {n_models} model(s) x " + f"{ens_result.persona_count} persona(s) x " + f"{ens_result.question_count} question(s)" + ) + print(f" Models: {', '.join(ensemble_models)}") + print(f" Errors: {len(incidents)} incident(s)") + print(f" Cost: {ens_result.total_cost.format_usd()} (recorded)") + if ensemble_result_id is not None: + print( + " Synthesis: skipped — ensemble runs never synthesize " + "automatically. Run: " + f"synthpanel panel synthesize {ensemble_result_id}" + ) + print(f" Results: {output['saved_path']}") + else: + print( + " Synthesis: skipped — ensemble runs never synthesize " + "automatically. Re-run with --save, then run: " + "synthpanel panel synthesize " + ) + print( + " Results: not saved — re-run with --save to persist " + "them (full per-model payload available via " + "--output-format json)" + ) + else: + emit(fmt, message="Ensemble complete", extra=output) if ensemble_result_id is not None: _print_saved_result_hint(ensemble_result_id) @@ -3106,6 +3166,31 @@ def _on_complete(pr: PanelistResult) -> None: hb_n = convergence_baseline_payload.get("converged_at") if hb_n is not None: print(f" human baseline: converged_at=n={hb_n}") + + # gh-r4: terse end-of-run summary so a default (text) run states + # panel size, error count, recorded cost, synthesis status, and + # where the results live — previously synthesis status and result + # location were only discoverable via --save/--output-format json. + if synthesis_dict: + synthesis_status = f"completed (cost: {synthesis_dict['cost']})" + elif getattr(args, "no_synthesis", False): + synthesis_status = "skipped (--no-synthesis)" + elif skip_synthesis: + synthesis_status = "skipped (run halted or invalid)" + else: + synthesis_status = "FAILED — see stderr above; recover with: synthpanel panel synthesize " + print(f"\n{'=' * 60}") + print("RUN SUMMARY") + print(f"{'=' * 60}") + print( + f" Personas: {len(personas)} Questions: {len(questions)} Errors: {failure_stats['errored_pairs']}" + ) + print(f" Cost: {total_cost_est.format_usd()} (recorded)") + print(f" Synthesis: {synthesis_status}") + if saved_result_path is not None: + print(f" Results: {saved_result_path}") + else: + print(" Results: not saved — re-run with --save to persist (then: synthpanel report )") else: # sp-efip: mirror the TEXT-mode behaviour and print the fatal # banner to stderr even when JSON/NDJSON is the primary output. @@ -6110,6 +6195,101 @@ def handle_cost_summary(args: argparse.Namespace, fmt: OutputFormat) -> int: return 0 +def handle_cost_show(args: argparse.Namespace, fmt: OutputFormat) -> int: + """Per-run cost breakdown for one saved result (gh-r4). + + Thin alias over the cost section of ``panel inspect`` — same result + resolution (ID or JSON path), same :class:`InspectReport` data, but + only the cost/usage fields. ``cost summary`` aggregates across runs; + this answers "what did run X cost?" without wading through the full + inspect output. + """ + from synth_panel.analysis.inspect import build_inspect_report + + result_ref = args.result + path = Path(result_ref) + if path.exists() and path.suffix == ".json": + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + msg = f"Error: not valid JSON: {result_ref}: {exc}" + if fmt is OutputFormat.TEXT: + print(msg, file=sys.stderr) + else: + emit(fmt, message=msg, extra={"error": "invalid_json"}) + return 1 + data.setdefault("id", path.stem) + else: + from synth_panel.mcp.data import get_panel_result + + try: + data = get_panel_result(result_ref) + except FileNotFoundError: + msg = f"Error: panel result not found: {result_ref}" + if fmt is OutputFormat.TEXT: + print(msg, file=sys.stderr) + else: + emit(fmt, message=msg, extra={"error": "not_found"}) + return 1 + + if not isinstance(data, dict): + msg = "Error: panel result is not a JSON object" + if fmt is OutputFormat.TEXT: + print(msg, file=sys.stderr) + else: + emit(fmt, message=msg, extra={"error": "invalid_shape"}) + return 1 + + report = build_inspect_report(data) + cost_payload: dict[str, Any] = { + "result_id": report.result_id, + "total_cost": report.total_cost, + "panelist_cost": report.panelist_cost, + "total_tokens": report.total_tokens, + "per_model": [ + { + "model": m.model, + "total_tokens": m.total_tokens, + "input_tokens": m.input_tokens, + "output_tokens": m.output_tokens, + "cost_usd": m.cost_usd, + } + for m in report.model_rollup + ], + "synthesis": { + "ran": report.synthesis.ran, + "model": report.synthesis.model, + "cost": report.synthesis.cost, + }, + } + + if fmt is OutputFormat.TEXT: + print(f"Cost for {report.result_id or result_ref}") + print(f" Total cost: {report.total_cost or '(not recorded)'}") + if report.panelist_cost: + print(f" Panelist cost: {report.panelist_cost}") + if report.synthesis.ran: + synth_line = report.synthesis.cost or "(not recorded)" + if report.synthesis.model: + synth_line += f" ({report.synthesis.model})" + print(f" Synthesis: {synth_line}") + else: + print(" Synthesis: not run") + print(f" Total tokens: {report.total_tokens}") + if report.model_rollup: + print(" Per model:") + for m in report.model_rollup: + cost_str = f"${m.cost_usd:.4f}" if m.cost_usd is not None else "(unknown)" + print( + f" {m.model}: {cost_str} " + f"(tokens={m.total_tokens} input={m.input_tokens} output={m.output_tokens})" + ) + else: + emit(fmt, message="Cost show", extra={"cost": cost_payload}) + + return 0 + + # --------------------------------------------------------------------------- # Instruments subcommands (sp-xsu) # --------------------------------------------------------------------------- @@ -7444,8 +7624,7 @@ def _print_saved_result_hint(result_id: str) -> None: print(f" synthpanel report {result_id} # full Markdown report (incl. cost rollup)", file=sys.stderr) print(f" synthpanel results show {result_id} # raw saved result", file=sys.stderr) print(" synthpanel results list # all saved results", file=sys.stderr) - # #538: there is no per-result `cost ` command — the per-run cost - # rollup lives in `report` above. `cost summary` aggregates across runs. + print(f" synthpanel cost show {result_id} # this run's cost breakdown", file=sys.stderr) print(" synthpanel cost summary # cost rollup across saved runs", file=sys.stderr) diff --git a/src/synth_panel/cli/parser.py b/src/synth_panel/cli/parser.py index 7fadb30..044e90d 100644 --- a/src/synth_panel/cli/parser.py +++ b/src/synth_panel/cli/parser.py @@ -138,11 +138,12 @@ def build_parser() -> argparse.ArgumentParser: panel_run_parser.add_argument( "--personas", default=None, - metavar="PATH", + metavar="NAME_OR_PATH", help=( - "Path to a YAML file defining personas. Required unless " - "--resume is given, in which case the original " - "personas path is recovered from the checkpoint." + "Name of a bundled/installed persona pack (see 'synthpanel pack " + "list', e.g. general-consumer) or a path to a YAML file defining " + "personas. Required unless --resume is given, in which " + "case the original personas path is recovered from the checkpoint." ), ) panel_run_parser.add_argument( @@ -173,11 +174,13 @@ def build_parser() -> argparse.ArgumentParser: panel_run_parser.add_argument( "--instrument", default=None, - metavar="PATH", + metavar="NAME_OR_PATH", help=( - "Path to a YAML file defining the survey/instrument. " - "Required unless --resume is given, in which case " - "the original instrument path is recovered from the checkpoint." + "Name of a bundled/installed instrument (see 'synthpanel " + "instruments list', e.g. general-survey) or a path to a YAML " + "file defining the survey/instrument. Required unless " + "--resume is given, in which case the original " + "instrument path is recovered from the checkpoint." ), ) panel_run_parser.add_argument( @@ -1216,6 +1219,16 @@ def build_parser() -> argparse.ArgumentParser: ) cost_subparsers = cost_parser.add_subparsers(dest="cost_command") + cost_show_parser = cost_subparsers.add_parser( + "show", + help="Per-run cost breakdown for one saved panel result (alias for the cost section of 'panel inspect').", + ) + cost_show_parser.add_argument( + "result", + metavar="RESULT", + help="Panel result ID or path to a result JSON file.", + ) + cost_summary_parser = cost_subparsers.add_parser( "summary", help="Summarize cost across saved panel runs (totals, by model, by month).", diff --git a/src/synth_panel/cost.py b/src/synth_panel/cost.py index 39c76c8..43644af 100644 --- a/src/synth_panel/cost.py +++ b/src/synth_panel/cost.py @@ -613,6 +613,25 @@ def resolve_cost( provider_total = float(usage.provider_reported_cost) local_total = local.total_cost + # gh-r4: a provider-reported cost of exactly $0 for a run that consumed + # tokens is almost never a real bill — OpenRouter returns ``usage.cost: + # 0`` for BYOK keys (the upstream provider bills directly) and some + # upstreams omit native cost. Trusting the zero verbatim recorded runs + # as $0.0000 and starved CostGate, so --max-cost could never trip. + # Fall back to the local pricing table whenever it prices the same + # usage as nonzero; genuinely free models (local table also prices $0) + # still record $0. + if provider_total == 0.0 and usage.total_tokens > 0 and local_total > 0: + logger.warning( + "Provider reported $0.00 for %d tokens on model=%s (BYOK or " + "missing native cost); recording the local pricing-table " + "estimate $%.6f instead.", + usage.total_tokens, + model, + local_total, + ) + return local + if local_total > 0 and provider_total > 0: ratio = abs(provider_total - local_total) / max(provider_total, local_total) if ratio > _DIVERGENCE_WARN_RATIO: diff --git a/src/synth_panel/main.py b/src/synth_panel/main.py index 7b59517..1e83277 100644 --- a/src/synth_panel/main.py +++ b/src/synth_panel/main.py @@ -215,6 +215,10 @@ def _main(argv: list[str] | None) -> int: sub = getattr(args, "cost_command", None) if sub == "summary": return handle_cost_summary(args, output_format) + elif sub == "show": + from synth_panel.cli.commands import handle_cost_show + + return handle_cost_show(args, output_format) else: parser.parse_args(["cost", "--help"]) return 1 diff --git a/tests/test_r4_cli_fixes.py b/tests/test_r4_cli_fixes.py new file mode 100644 index 0000000..1f68a1c --- /dev/null +++ b/tests/test_r4_cli_fixes.py @@ -0,0 +1,331 @@ +"""Regression tests for the gh-r4 naive-user CLI fixes. + +Covers: + +* Routing — a one-entry weight-free ``--models`` spec is a standard + single-model panel run (synthesis included), not an "ensemble" that + silently skips synthesis and exits 0 with a bare "Ensemble complete". +* Cost — a provider-reported cost of exactly $0 for a run that consumed + tokens (OpenRouter BYOK) must not zero out the recorded run cost or + starve :class:`~synth_panel.cost.CostGate`; the local pricing-table + estimate is used instead. +* Ensemble stdout — genuine (>=2 model) ensemble runs now state on stdout + that synthesis was skipped and print the exact follow-up command. +* ``cost show `` — per-run cost breakdown alias over the + inspect data. +* Help text — ``--personas`` / ``--instrument`` document bundled + pack/instrument names, not only YAML paths. +""" + +from __future__ import annotations + +import json +from decimal import Decimal +from unittest.mock import MagicMock, patch + +import pytest + +from synth_panel.cost import CostGate, TokenUsage, resolve_cost +from synth_panel.main import main +from synth_panel.persistence import ConversationMessage +from synth_panel.runtime import TurnSummary + + +def _mock_turn_summary(text: str = "I think so.") -> TurnSummary: + usage = TokenUsage(input_tokens=1000, output_tokens=200) + msg = ConversationMessage( + role="assistant", + content=[{"type": "text", "text": text}], + usage=usage, + ) + return TurnSummary(assistant_messages=[msg], iterations=1, usage=usage) + + +def _mock_synthesis_result(): + from synth_panel.cost import CostEstimate + from synth_panel.synthesis import SynthesisResult + + return SynthesisResult( + summary="Test synthesis summary", + themes=["theme1"], + agreements=[], + disagreements=[], + surprises=[], + recommendation="Do X", + usage=TokenUsage(input_tokens=100, output_tokens=50), + cost=CostEstimate(), + model="modelA", + ) + + +@pytest.fixture +def panel_files(tmp_path): + personas_file = tmp_path / "personas.yaml" + personas_file.write_text("personas:\n - name: Alice\n age: 30\n - name: Bob\n age: 40\n") + survey_file = tmp_path / "survey.yaml" + survey_file.write_text("instrument:\n questions:\n - text: What do you think?\n") + return personas_file, survey_file + + +class TestSingleModelsSpecRouting: + """A one-entry ``--models`` spec must take the standard single-model path.""" + + @patch("synth_panel.cli.commands.synthesize_panel") + @patch("synth_panel.orchestrator.AgentRuntime") + @patch("synth_panel.cli.commands.LLMClient") + def test_single_entry_models_runs_synthesis( + self, mock_client_cls, mock_runtime_cls, mock_synth, capsys, panel_files + ): + mock_runtime = MagicMock() + mock_runtime.run_turn.return_value = _mock_turn_summary() + mock_runtime_cls.return_value = mock_runtime + mock_synth.return_value = _mock_synthesis_result() + personas_file, survey_file = panel_files + + code = main( + [ + "panel", + "run", + "--models", + "modelA", + "--personas", + str(personas_file), + "--instrument", + str(survey_file), + ] + ) + assert code == 0 + captured = capsys.readouterr() + # Synthesis ran (it never did on the old ensemble path). + mock_synth.assert_called_once() + assert "SYNTHESIS" in captured.out + assert "Test synthesis summary" in captured.out + # No "Ensemble" phrasing for a single-model run. + assert "Ensemble complete" not in captured.out + # The demotion is announced. + assert "single model" in captured.err + # End-of-run summary states synthesis status + where results live. + assert "RUN SUMMARY" in captured.out + assert "Synthesis: completed" in captured.out + assert "not saved" in captured.out + + @patch("synth_panel.cli.commands.synthesize_panel") + @patch("synth_panel.orchestrator.AgentRuntime") + @patch("synth_panel.cli.commands.LLMClient") + def test_single_entry_models_json_matches_single_model_shape( + self, mock_client_cls, mock_runtime_cls, mock_synth, capsys, panel_files + ): + mock_runtime = MagicMock() + mock_runtime.run_turn.return_value = _mock_turn_summary() + mock_runtime_cls.return_value = mock_runtime + mock_synth.return_value = _mock_synthesis_result() + personas_file, survey_file = panel_files + + code = main( + [ + "--output-format", + "json", + "panel", + "run", + "--models", + "modelA", + "--personas", + str(personas_file), + "--instrument", + str(survey_file), + ] + ) + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["persona_count"] == 2 + assert data["synthesis"] is not None + assert data["synthesis"]["summary"] == "Test synthesis summary" + # Nonzero recorded cost with token usage (regression for gh-r4 #2). + assert data["panelist_cost"] not in (None, "$0.0000") + + @patch("synth_panel.orchestrator.AgentRuntime") + @patch("synth_panel.cli.commands.LLMClient") + def test_two_model_ensemble_states_synthesis_skipped(self, mock_client_cls, mock_runtime_cls, capsys, panel_files): + mock_runtime = MagicMock() + mock_runtime.run_turn.return_value = _mock_turn_summary() + mock_runtime_cls.return_value = mock_runtime + personas_file, survey_file = panel_files + + code = main( + [ + "panel", + "run", + "--models", + "modelA,modelB", + "--skip-preflight", + "--personas", + str(personas_file), + "--instrument", + str(survey_file), + ] + ) + assert code == 0 + out = capsys.readouterr().out + assert "Ensemble complete" in out + assert "Synthesis: skipped" in out + assert "panel synthesize" in out + # Recorded cost + panel dimensions are on stdout, not hidden. + assert "Cost:" in out + assert "2 model(s)" in out + + @patch("synth_panel.orchestrator.AgentRuntime") + @patch("synth_panel.cli.commands.LLMClient") + def test_two_model_ensemble_json_carries_synthesis_status( + self, mock_client_cls, mock_runtime_cls, capsys, panel_files + ): + mock_runtime = MagicMock() + mock_runtime.run_turn.return_value = _mock_turn_summary() + mock_runtime_cls.return_value = mock_runtime + personas_file, survey_file = panel_files + + code = main( + [ + "--output-format", + "json", + "panel", + "run", + "--models", + "modelA,modelB", + "--skip-preflight", + "--personas", + str(personas_file), + "--instrument", + str(survey_file), + ] + ) + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["synthesis"] is None + assert data["synthesis_status"] == "skipped" + + +class TestZeroProviderCostFallback: + """resolve_cost must not record $0 for token-consuming runs (gh-r4 #2).""" + + MODEL = "openrouter/google/gemini-2.5-flash-lite" + + def test_zero_provider_cost_with_tokens_uses_local_estimate(self): + usage = TokenUsage(input_tokens=33_368, output_tokens=7_590, provider_reported_cost=0) + cost = resolve_cost(usage, self.MODEL) + assert cost.total_cost > 0 + + def test_nonzero_provider_cost_still_wins(self): + usage = TokenUsage(input_tokens=33_368, output_tokens=7_590, provider_reported_cost=Decimal("0.0041")) + cost = resolve_cost(usage, self.MODEL) + assert cost.total_cost == pytest.approx(0.0041) + + def test_zero_provider_cost_without_tokens_stays_zero(self): + usage = TokenUsage(input_tokens=0, output_tokens=0, provider_reported_cost=0) + cost = resolve_cost(usage, self.MODEL) + assert cost.total_cost == 0 + + def test_cost_gate_sees_accrued_cost_despite_zero_provider_cost(self): + """CostGate must accrue nonzero spend so --max-cost can trip.""" + from synth_panel.orchestrator import run_panel_parallel + + personas = [{"name": f"P{i}"} for i in range(4)] + questions = [{"text": "Q?"}] + + usage = TokenUsage(input_tokens=50_000, output_tokens=10_000, provider_reported_cost=0) + msg = ConversationMessage(role="assistant", content=[{"type": "text", "text": "ok"}], usage=usage) + summary = TurnSummary(assistant_messages=[msg], iterations=1, usage=usage) + + gate = CostGate(max_cost_usd=0.001, total_panelists=len(personas)) + with patch("synth_panel.orchestrator.AgentRuntime") as mock_runtime_cls: + mock_runtime = MagicMock() + mock_runtime.run_turn.return_value = summary + mock_runtime_cls.return_value = mock_runtime + run_panel_parallel( + client=MagicMock(), + personas=personas, + questions=questions, + model=self.MODEL, + system_prompt_fn=lambda p: "system", + question_prompt_fn=lambda q: q["text"], + cost_gate=gate, + max_workers=1, + ) + + snap = gate.snapshot() + assert snap["running_cost_usd"] > 0 + # 60k tokens on a priced model projects far beyond a $0.001 cap. + assert snap["halted"] is True + + +class TestCostShow: + """`cost show ` surfaces the per-run cost breakdown.""" + + def _write_result(self, tmp_path): + data = { + "results": [ + { + "persona": "Alice", + "model": "modelA", + "responses": [{"question": "Q?", "response": "A."}], + "usage": {"input_tokens": 100, "output_tokens": 20}, + } + ], + "model": "modelA", + "total_cost": "$1.2345", + "panelist_cost": "$1.2000", + "total_usage": {"input_tokens": 100, "output_tokens": 20}, + "persona_count": 1, + "question_count": 1, + } + p = tmp_path / "result-test.json" + p.write_text(json.dumps(data)) + return p + + def test_cost_show_text(self, capsys, tmp_path): + p = self._write_result(tmp_path) + code = main(["cost", "show", str(p)]) + assert code == 0 + out = capsys.readouterr().out + assert "Total cost: $1.2345" in out + assert "Panelist cost: $1.2000" in out + assert "Synthesis: not run" in out + + def test_cost_show_json(self, capsys, tmp_path): + p = self._write_result(tmp_path) + code = main(["--output-format", "json", "cost", "show", str(p)]) + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["cost"]["total_cost"] == "$1.2345" + assert data["cost"]["synthesis"]["ran"] is False + + def test_cost_show_missing_result(self, capsys): + code = main(["cost", "show", "result-does-not-exist"]) + assert code == 1 + assert "not found" in capsys.readouterr().err + + +class TestHelpText: + def test_personas_help_mentions_bundled_packs(self): + from synth_panel.cli.parser import build_parser + + parser = build_parser() + # Find the panel run subparser's help text. + import argparse + + def _find(parser, names): + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for name, sub in action.choices.items(): + if name == names[0]: + if len(names) == 1: + return sub + return _find(sub, names[1:]) + return None + + run_parser = _find(parser, ["panel", "run"]) + assert run_parser is not None + helps = {a.option_strings[0]: (a.help or "") for a in run_parser._actions if a.option_strings} + assert "pack list" in helps["--personas"] + assert "YAML" in helps["--personas"] + assert "instruments list" in helps["--instrument"] + assert "YAML" in helps["--instrument"] From 16d8c1af4336a4e6798a46507897049dc5bead3e Mon Sep 17 00:00:00 2001 From: OpenClaw Date: Fri, 17 Jul 2026 00:13:37 -0500 Subject: [PATCH 2/2] fix(cli): narrow model_spec for mypy after single-entry --models demotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI typecheck flagged commands.py:1731 — the demotion branch nulls model_spec together with has_models, but mypy cannot see the invariant across the two blocks. Assert it explicitly. Co-Authored-By: Claude Fable 5 --- src/synth_panel/cli/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/synth_panel/cli/commands.py b/src/synth_panel/cli/commands.py index f137f6e..5ece62b 100644 --- a/src/synth_panel/cli/commands.py +++ b/src/synth_panel/cli/commands.py @@ -1710,6 +1710,9 @@ def system_prompt_fn(persona: dict) -> str: file=sys.stderr, ) if has_models: + # The demotion above nulls has_models and model_spec together, so a + # truthy has_models implies a parsed spec; assert for mypy's sake. + assert model_spec is not None # sp-zdul: warn if weighted sum is far from 1.0 — the user likely # intended an exact-ratio split and a typo (e.g. "0.3,0.3,0.3") # silently reweights their panel.