diff --git a/docs/production-operations.md b/docs/production-operations.md index f23b892..85a8c58 100644 --- a/docs/production-operations.md +++ b/docs/production-operations.md @@ -82,9 +82,18 @@ per-persona error strings, so the banner names the actual upstream failure |---|---| | `0` | Run completed and is valid. | | `1` | Startup/config error (bad flags, missing files, refused flag combos). | -| `2` | Run completed but **invalid** (`run_invalid: true`): failure rate over threshold, total failure, cost gate tripped, SIGINT, missing-input refusals, or synthesis failure. | +| `2` | Run completed but **invalid** (`run_invalid: true`): failure rate over threshold, total failure, cost gate tripped, SIGINT, or missing-input refusals. | | `3` | `--strict` violation: any panelist-question error at all. | +A **synthesis-stage failure does not invalidate the run**: when every +panelist response completed, the envelope keeps them, sets +`synthesis: null` (or the fallback synthesis payload), carries a +structured top-level `synthesis_error` plus a `synthesis_failed: ...` +warning, and exits `0`. Recover the missing synthesis cheaply with +`synthpanel panel synthesize ` instead of re-running the +panel. (`panel synthesize` itself still exits `2` on failure — synthesis +is that command's sole deliverable.) + **Every abort path still emits valid partial JSON.** With `--output-format json`, a cost-gate halt, SIGINT, or total failure does not produce truncated garbage — it produces a complete, parseable envelope with diff --git a/src/synth_panel/_runners.py b/src/synth_panel/_runners.py index 0864cb3..cebf073 100644 --- a/src/synth_panel/_runners.py +++ b/src/synth_panel/_runners.py @@ -786,8 +786,9 @@ def run_panel_sync( synthesis_dict: dict[str, Any] | None = None # sp-avmm: synthesis failures surface as a ``synthesis_error`` key on # ``synthesis_dict``. Callers (MCP server / SDK) detect this and lift - # it to ``run_invalid=True`` + top-level ``synthesis_error`` on the - # result envelope. + # it to a top-level ``synthesis_error`` + warning on the result + # envelope. It does not invalidate the run — panelist responses are + # complete; consumers can re-synthesize the saved result. # GH#576: mirror the CLI's cost-gate discipline — a halted run is a # deliberately-truncated partial, so spending more budget synthesizing # it would produce an untrustworthy result. Skip synthesis entirely. diff --git a/src/synth_panel/cli/commands.py b/src/synth_panel/cli/commands.py index 5ece62b..5f712e3 100644 --- a/src/synth_panel/cli/commands.py +++ b/src/synth_panel/cli/commands.py @@ -2715,7 +2715,6 @@ def _on_complete(pr: PanelistResult) -> None: ), diagnostic=overflow, ) - run_invalid = True else: try: if resolved_strategy == STRATEGY_MAP_REDUCE: @@ -2767,7 +2766,6 @@ def _on_complete(pr: PanelistResult) -> None: ), diagnostic=exc.diagnostic, ) - run_invalid = True except Exception as exc: # sp-rcvr: synthesis recovery ladder — classify the failure, # retry transients once, fall back to map-reduce on (suspected) @@ -2828,14 +2826,12 @@ def _on_complete(pr: PanelistResult) -> None: suggested_fix=suggested, diagnostic=diagnostic, ) - run_invalid = True # sy-549: a synthesis that returned a *fallback* result (judge exhausted # its schema-adherence retries, or returned a partial schema) is NOT a # healthy synthesis. Previously such a result was persisted as if it # succeeded, hiding the failure behind mostly-empty fields. Surface a - # loud warning + a machine-detectable error payload and mark the run - # invalid, mirroring the sp-avmm hard-fail on synthesis API errors. + # loud warning + a machine-detectable error payload. if synthesis_result is not None and getattr(synthesis_result, "is_fallback", False): synth_err = synthesis_result.error or "synthesis judge did not return a valid structured result" logger.warning("synthesis produced a fallback result: %s", synth_err) @@ -2850,7 +2846,6 @@ def _on_complete(pr: PanelistResult) -> None: "result with `synthpanel panel synthesize --synthesis-model ...`." ), ) - run_invalid = True if synthesis_result: total_usage = panelist_usage + synthesis_result.usage @@ -3313,7 +3308,19 @@ def _cli_panelist_formatter(pr: PanelistResult, panel_model: str) -> dict[str, A if synthesis_error_payload is not None: # sp-avmm: top-level synthesis_error so JSON/NDJSON consumers # see the failure without walking into the synthesis dict. + # A synthesis-stage failure does NOT invalidate the run — the + # panelist responses above are complete and usable. Degrade to + # synthesis: null/fallback + a warning; recovery is a cheap + # re-synthesize of the saved result, not a full re-run. extra["synthesis_error"] = synthesis_error_payload + warnings_list = extra.setdefault("warnings", []) + if isinstance(warnings_list, list): + warnings_list.append( + "synthesis_failed: " + f"{synthesis_error_payload.get('message', 'synthesis failed')}" + " — panelist responses are complete; recover with" + " `synthpanel panel synthesize `" + ) if missing_input_invalid: # sp-bjt4: surface as a top-level warning so MCP / CI consumers # see the condition even if they don't special-case @@ -3702,7 +3709,6 @@ def _handle_panel_run_multi_round( "Rerun with a higher-quality --synthesis-model." ), ) - run_invalid = True # ── Cost / usage aggregation ──────────────────────────────────────── panelist_usage = ZERO_USAGE @@ -3940,7 +3946,17 @@ def _handle_panel_run_multi_round( extra["total_failure"] = total_failure extra["abort_reason"] = _classify_total_failure_abort_reason(total_failure) if synthesis_error_payload is not None: + # Synthesis-stage failure degrades gracefully: the per-round + # panelist responses are complete, so surface the payload plus + # a warning instead of invalidating the run. extra["synthesis_error"] = synthesis_error_payload + warnings_list = extra.setdefault("warnings", []) + if isinstance(warnings_list, list): + warnings_list.append( + "synthesis_failed: " + f"{synthesis_error_payload.get('message', 'synthesis failed')}" + " — panelist responses are complete" + ) if missing_input_invalid: warnings_list = extra.setdefault("warnings", []) if isinstance(warnings_list, list): diff --git a/src/synth_panel/cost.py b/src/synth_panel/cost.py index 43644af..dd50eba 100644 --- a/src/synth_panel/cost.py +++ b/src/synth_panel/cost.py @@ -329,14 +329,14 @@ class ModelPricing: ) # Meta Llama 3.3 70B Instruct (OpenRouter: ``meta-llama/llama-3.3-70b-instruct``). -# OR feed (2026-05-10): $0.10/M input, $0.32/M output. Prior rates -# ($0.23/$0.40) overstated cost ~2x — OR's blended provider rate has -# dropped as more low-cost providers came online. +# OR feed (2026-07-19): $0.13/M input, $0.40/M output. The 2026-05-10 +# snapshot ($0.10/$0.32) understated provider-reported cost ~30% on live +# runs — OR's blended provider rate moved back up. LLAMA_3_3_70B_PRICING = ModelPricing( - input_cost_per_million=0.10, - output_cost_per_million=0.32, - cache_creation_cost_per_million=0.10, - cache_read_cost_per_million=0.10, + input_cost_per_million=0.13, + output_cost_per_million=0.40, + cache_creation_cost_per_million=0.13, + cache_read_cost_per_million=0.13, ) DEFAULT_PRICING = SONNET_PRICING diff --git a/src/synth_panel/mcp/server.py b/src/synth_panel/mcp/server.py index cd0f577..1ab6eec 100644 --- a/src/synth_panel/mcp/server.py +++ b/src/synth_panel/mcp/server.py @@ -1568,11 +1568,17 @@ async def _run_panel_async( # sp-avmm: synthesis failure must surface loudly at the envelope # top-level. Without this, MCP callers see synthesis: {synthesis_error: - # …} buried inside the result and cannot gate on run validity without - # inspecting the nested payload. + # …} buried inside the result and cannot spot the failure without + # inspecting the nested payload. It does NOT set run_invalid — the + # panelist responses are complete and usable; degrade to a warning so + # callers can re-synthesize instead of discarding the run. if synthesis_dict and isinstance(synthesis_dict.get("synthesis_error"), dict): - result["run_invalid"] = True result["synthesis_error"] = synthesis_dict["synthesis_error"] + result["warnings"].append( + "synthesis_failed: " + f"{synthesis_dict['synthesis_error'].get('message', 'synthesis failed')}" + " — panelist responses are complete" + ) # GH#576: cost gate tripped mid-run — the orchestrator cancelled the # pending panelists and ``result_dicts`` is a valid completed prefix. diff --git a/src/synth_panel/sdk.py b/src/synth_panel/sdk.py index 7847f00..260f40e 100644 --- a/src/synth_panel/sdk.py +++ b/src/synth_panel/sdk.py @@ -240,8 +240,10 @@ class PollResult(_DictLikeMixin): total_usage: dict[str, Any] total_cost: str metadata: dict[str, Any] | None = None - # sp-avmm: synthesis failure surfaced at the top-level so consumers can - # gate on validity without walking into the nested synthesis dict. + # sp-avmm: synthesis failure surfaced at the top-level (synthesis_error) + # so consumers see it without walking into the nested synthesis dict. + # run_invalid stays False for synthesis-stage failures — panelist + # responses are complete; it flags panelist-level invalidity only. run_invalid: bool = False synthesis_error: dict[str, Any] | None = None # sy-4yd: deterministic structured-response rollup (vote counts, @@ -311,7 +313,8 @@ class PanelResult(_DictLikeMixin): terminal_round: str | None = None results: list[dict[str, Any]] = field(default_factory=list) metadata: dict[str, Any] | None = None - # sp-avmm: synthesis failure markers, mirroring PollResult. + # sp-avmm: synthesis failure markers, mirroring PollResult. run_invalid + # is not set by synthesis-stage failures (see PollResult). run_invalid: bool = False synthesis_error: dict[str, Any] | None = None # sy-4yd: deterministic structured-response rollup. See PollResult @@ -543,13 +546,21 @@ def _build_panel_result_from_single_round( # otherwise the single ``model`` is the only candidate. cost_warnings = build_cost_fallback_warnings(contributing_models if contributing_models is not None else [model]) # sp-avmm: carry synthesis failure markers onto the top-level result - # so MCP/CI consumers can branch on run validity without walking into - # the nested synthesis envelope. + # so MCP/CI consumers can spot the failure without walking into the + # nested synthesis envelope. A synthesis-stage failure does NOT set + # run_invalid — the panelist responses are complete and usable, so + # degrade to a warning + synthesis_error instead of invalidating. synthesis_error = synthesis_dict.get("synthesis_error") if isinstance(synthesis_dict, dict) else None # sp-g59o: surface synthesis-level heuristic warnings (e.g. degenerate # structured output) at the top level so MCP consumers don't have to # walk into the nested synthesis dict to find them. synthesis_warnings = list(synthesis_dict.get("warnings") or []) if isinstance(synthesis_dict, dict) else [] + if isinstance(synthesis_error, dict): + synthesis_warnings.append( + "synthesis_failed: " + f"{synthesis_error.get('message', 'synthesis failed')}" + " — panelist responses are complete" + ) poll_summary_payload = _compute_poll_summary_payload( result_dicts=result_dicts, questions=questions, @@ -577,7 +588,7 @@ def _build_panel_result_from_single_round( terminal_round=None, results=result_dicts, metadata=metadata, - run_invalid=bool(synthesis_error), + run_invalid=False, synthesis_error=synthesis_error if isinstance(synthesis_error, dict) else None, poll_summary=poll_summary_payload, ) @@ -896,7 +907,10 @@ def quick_poll( total_usage=total_usage.to_dict(), total_cost=total_cost.format_usd(), metadata=metadata, - run_invalid=bool(synthesis_error), + # A synthesis-stage failure does not invalidate the poll — the + # panelist responses are complete; synthesis_error carries the + # failure detail for consumers that want to re-synthesize. + run_invalid=False, synthesis_error=synthesis_error if isinstance(synthesis_error, dict) else None, poll_summary=poll_summary_payload, ) diff --git a/src/synth_panel/structured/models.py b/src/synth_panel/structured/models.py index 99603af..2d5563f 100644 --- a/src/synth_panel/structured/models.py +++ b/src/synth_panel/structured/models.py @@ -14,9 +14,10 @@ from __future__ import annotations -from typing import Annotated +import re +from typing import Annotated, Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class RankedItem(BaseModel): @@ -82,6 +83,11 @@ class AnnotatedChoice(BaseModel): attachment_id: str = "" +# Leading bullet / enumeration markers stripped when coercing a +# newline-bulleted string into list items: "-", "*", "•", "–", "1.", "1)". +_BULLET_PREFIX_RE = re.compile(r"^\s*(?:[-*•–]|\d+[.)])\s+") + + class PartialSummary(BaseModel): """Per-question map-phase synthesis partial (v1.0.3 P2). @@ -91,6 +97,12 @@ class PartialSummary(BaseModel): single map call surfaces as a :class:`pydantic.ValidationError` instead of silently propagating empty themes through to the final synthesis. + + List fields tolerate a string value by splitting it into items + (newline-bulleted lists like ``"\\n- A\\n- B"`` are the observed + provider drift). The single-pass synthesis path never re-validates + these fields, so a string slips through it unchanged — the map + boundary must not be stricter than single-pass for the same drift. """ summary: str @@ -100,6 +112,14 @@ class PartialSummary(BaseModel): surprises: list[str] recommendation: str + @field_validator("themes", "agreements", "disagreements", "surprises", mode="before") + @classmethod + def _coerce_string_to_list(cls, v: Any) -> Any: + if not isinstance(v, str): + return v + items = [_BULLET_PREFIX_RE.sub("", line).strip() for line in v.splitlines()] + return [item for item in items if item] + MODEL_REGISTRY: dict[str, type[BaseModel]] = { "ranking": Ranking, diff --git a/src/synth_panel/synthesis.py b/src/synth_panel/synthesis.py index 5e6fcc8..ce2b9dc 100644 --- a/src/synth_panel/synthesis.py +++ b/src/synth_panel/synthesis.py @@ -1421,17 +1421,15 @@ def _run_one_map(idx: int) -> tuple[int, SynthesisResult, dict[str, Any]]: is_fallback=True, ) try: - typed_partials.append( - PartialSummary.model_validate( - { - "summary": res.summary, - "themes": res.themes, - "agreements": res.agreements, - "disagreements": res.disagreements, - "surprises": res.surprises, - "recommendation": res.recommendation, - } - ) + partial = PartialSummary.model_validate( + { + "summary": res.summary, + "themes": res.themes, + "agreements": res.agreements, + "disagreements": res.disagreements, + "surprises": res.surprises, + "recommendation": res.recommendation, + } ) except _PydanticValidationError as ve: raise MapPhaseFailure( @@ -1439,6 +1437,14 @@ def _run_one_map(idx: int) -> tuple[int, SynthesisResult, dict[str, Any]]: question_index=i, validation_error=ve, ) from ve + typed_partials.append(partial) + # Write the coerced list fields back so the reduce stage and the + # per_question rollup see lists even when the map model emitted + # newline-bulleted strings (the drift PartialSummary tolerates). + res.themes = partial.themes + res.agreements = partial.agreements + res.disagreements = partial.disagreements + res.surprises = partial.surprises # Reduce phase synthetic_panelists = _build_synthetic_reduce_panelists(completed_maps, questions) diff --git a/tests/test_synthesis_failure.py b/tests/test_synthesis_failure.py index 4cd6180..1944715 100644 --- a/tests/test_synthesis_failure.py +++ b/tests/test_synthesis_failure.py @@ -1,18 +1,19 @@ -"""sp-avmm: synthesis failure must fail loud, not silently return success. +"""sp-avmm: synthesis failure must surface loudly — without invalidating +an otherwise-complete run. Covers the three call sites the bead calls out: -* ``cmd_panel_run`` (non-ensemble CLI path) — previously caught the - exception, logged a WARN, then emitted a result with ``synthesis: null`` - and exit 0. Now: pre-flight context check + structured ``synthesis_error`` - + ``run_invalid: true`` + exit 2. -* ``handle_panel_synthesize`` (re-synthesize saved result) — previously - exited 1 with a bare stderr line. Now: exit 2 and a structured envelope - in JSON/NDJSON mode so MCP/CI consumers see the same shape. -* ``run_panel_sync`` (MCP/SDK sync runner) — previously set - ``{"synthesis_error": "Synthesis failed — see logs for details."}`` - without flagging ``run_invalid``. Now: structured payload and, via the - server/sdk envelope, ``run_invalid=True``. +* ``cmd_panel_run`` (non-ensemble CLI path) — a synthesis-stage failure + emits a structured ``synthesis_error`` + a ``synthesis_failed`` warning + but keeps ``run_invalid: false`` and exits 0: every panelist response + completed and is present in the output, so the run degrades to + ``synthesis: null`` instead of being discarded. +* ``handle_panel_synthesize`` (re-synthesize saved result) — synthesis IS + the command's sole deliverable, so failure there still exits 2 with a + structured envelope in JSON/NDJSON mode. +* ``run_panel_sync`` (MCP/SDK sync runner) — surfaces the structured + ``synthesis_error`` payload; the server/sdk envelope carries it at the + top level as a warning, not as ``run_invalid``. """ from __future__ import annotations @@ -64,9 +65,13 @@ class TestCliNonEnsembleSynthesisFailure: @patch("synth_panel.cli.commands.synthesize_panel") @patch("synth_panel.cli.commands.run_panel_parallel") @patch("synth_panel.cli.commands.LLMClient") - def test_synthesis_api_error_sets_run_invalid(self, _mock_client, mock_run, mock_synth, capsys, tmp_path): - """When synthesize_panel raises an API error, the run must exit 2, - set run_invalid=True, and carry a structured synthesis_error payload. + def test_synthesis_api_error_degrades_gracefully(self, _mock_client, mock_run, mock_synth, capsys, tmp_path): + """When synthesize_panel raises an API error but every panelist + completed, the run must exit 0 with run_invalid=False, keep the + panelist responses in the output, carry a structured + synthesis_error payload, and surface a synthesis_failed warning — + NOT invalidate the whole run (regression: synthesis-stage failure + used to exit 2 with message="PANEL RUN INVALID"). """ from synth_panel.orchestrator import WorkerRegistry @@ -85,18 +90,25 @@ def test_synthesis_api_error_sets_run_invalid(self, _mock_client, mock_run, mock _write_instrument(tmp_path / "survey.yaml"), ] ) - assert code == 2, "sp-avmm requires exit code 2 on synthesis failure" + assert code == 0, "synthesis failure on a complete panel must not fail the run" captured = capsys.readouterr() assert "synthesis call failed" in captured.err.lower() assert "262000" in captured.err or "262k" in captured.err or "200000" in captured.err payload = json.loads(captured.out) - assert payload["run_invalid"] is True + assert payload["run_invalid"] is False + assert payload.get("message") != "PANEL RUN INVALID" + # Panelist responses survived the synthesis failure. + flat = payload["rounds"][0]["results"] + assert len(flat) == 2 + assert all(r["responses"] for r in flat) + assert payload["synthesis"] is None err = payload["synthesis_error"] assert isinstance(err, dict), "synthesis_error must be a structured dict" assert err["error_type"] == "synthesis_api_error" assert "message" in err assert "suggested_fix" in err + assert any("synthesis_failed" in w for w in payload["warnings"]) mock_synth.assert_called_once() @patch("synth_panel.cli.commands.synthesize_panel") @@ -141,18 +153,20 @@ def test_preflight_rejects_oversized_input(self, _mock_client, mock_run, mock_sy "single", ] ) - assert code == 2 + assert code == 0, "pre-flight rejection must not invalidate a complete panel" captured = capsys.readouterr() assert "pre-flight" in captured.err.lower() or "exceeds" in captured.err.lower() mock_synth.assert_not_called(), "pre-flight must short-circuit BEFORE the API call" payload = json.loads(captured.out) - assert payload["run_invalid"] is True + assert payload["run_invalid"] is False + assert payload["synthesis"] is None err = payload["synthesis_error"] assert err["error_type"] == "synthesis_context_overflow" diag = err["diagnostic"] assert diag["context_window"] == 200_000 assert diag["estimated_tokens"] > diag["effective_limit"] + assert any("synthesis_failed" in w for w in payload["warnings"]) # --- sp-avmm: handle_panel_synthesize (re-synthesize) path ---------------- @@ -451,7 +465,8 @@ def fake_post(url, **kwargs): class TestSynthesisFallbackSurfaced: """sy-549 (2): a fallback synthesis (judge exhausted retries / partial schema) must not be persisted as a silent success — it carries an - ``is_fallback`` marker in to_dict() and fails the run loudly.""" + ``is_fallback`` marker in to_dict() and surfaces a structured + synthesis_error + warning (without invalidating the completed panel).""" def test_to_dict_marks_fallback(self): from synth_panel.synthesis import SynthesisResult @@ -488,7 +503,7 @@ def test_healthy_synthesis_has_no_fallback_marker(self): @patch("synth_panel.cli.commands.synthesize_panel") @patch("synth_panel.cli.commands.run_panel_parallel") @patch("synth_panel.cli.commands.LLMClient") - def test_inrun_fallback_sets_run_invalid(self, _mock_client, mock_run, mock_synth, capsys, tmp_path): + def test_inrun_fallback_degrades_gracefully(self, _mock_client, mock_run, mock_synth, capsys, tmp_path): from synth_panel.orchestrator import WorkerRegistry from synth_panel.synthesis import SynthesisResult @@ -516,13 +531,14 @@ def test_inrun_fallback_sets_run_invalid(self, _mock_client, mock_run, mock_synt _write_instrument(tmp_path / "survey.yaml"), ] ) - assert code == 2 + assert code == 0, "fallback synthesis on a complete panel must not fail the run" captured = capsys.readouterr() assert "incomplete" in captured.err.lower() payload = json.loads(captured.out) - assert payload["run_invalid"] is True + assert payload["run_invalid"] is False err = payload["synthesis_error"] assert err["error_type"] == "synthesis_fallback" + assert any("synthesis_failed" in w for w in payload["warnings"]) @patch("synth_panel.cli.commands.synthesize_panel") def test_resynthesize_fallback_exits_two(self, mock_synth, capsys, tmp_path): diff --git a/tests/test_synthesis_strategy_interaction.py b/tests/test_synthesis_strategy_interaction.py index 53929f4..99b3e55 100644 --- a/tests/test_synthesis_strategy_interaction.py +++ b/tests/test_synthesis_strategy_interaction.py @@ -176,13 +176,15 @@ def test_single_still_rejects_on_overflow( ] ) - assert code == 2 + assert code == 0, "pre-flight rejection degrades gracefully; panel responses are complete" captured = capsys.readouterr() assert "pre-flight" in captured.err.lower() or "exceeds" in captured.err.lower() mock_synth_single.assert_not_called() mock_synth_mr.assert_not_called() payload = json.loads(captured.out) + assert payload["run_invalid"] is False + assert payload["synthesis"] is None err = payload["synthesis_error"] assert err["error_type"] == "synthesis_context_overflow" @@ -367,8 +369,9 @@ def test_per_chunk_overflow_becomes_structured_payload( ] ) - assert code == 2 + assert code == 0, "chunk overflow degrades gracefully; panel responses are complete" payload = json.loads(capsys.readouterr().out) + assert payload["run_invalid"] is False err = payload["synthesis_error"] assert err["error_type"] == "synthesis_map_chunk_overflow" assert err["diagnostic"]["question_index"] == 0 diff --git a/tests/test_synthesis_typed_partials.py b/tests/test_synthesis_typed_partials.py index 6150208..f79d59e 100644 --- a/tests/test_synthesis_typed_partials.py +++ b/tests/test_synthesis_typed_partials.py @@ -343,3 +343,117 @@ def test_missing_required_field_raises(self): "recommendation": "r", } ) + + +class TestPartialSummaryStringCoercion: + """Regression (2026-07-19 live repro): map models sometimes emit the + list fields as ONE newline-bulleted string ("\\n- A\\n- B") instead of + a JSON array. The single-pass path tolerates that drift, so the map + boundary must coerce it rather than fail typed validation.""" + + def test_newline_bulleted_string_coerces_to_list(self): + p = PartialSummary.model_validate( + { + "summary": "s", + "themes": "\n- Distinctiveness and identity\n- Pricing anxiety", + "agreements": "- All want speed\n- All dislike ads", + "disagreements": "* One split\n* Another split", + "surprises": "1. First surprise\n2) Second surprise", + "recommendation": "r", + } + ) + assert p.themes == ["Distinctiveness and identity", "Pricing anxiety"] + assert p.agreements == ["All want speed", "All dislike ads"] + assert p.disagreements == ["One split", "Another split"] + assert p.surprises == ["First surprise", "Second surprise"] + + def test_plain_single_line_string_becomes_one_item(self): + p = PartialSummary.model_validate( + { + "summary": "s", + "themes": "just one theme", + "agreements": [], + "disagreements": [], + "surprises": [], + "recommendation": "r", + } + ) + assert p.themes == ["just one theme"] + + def test_empty_string_becomes_empty_list(self): + p = PartialSummary.model_validate( + { + "summary": "s", + "themes": "", + "agreements": " \n ", + "disagreements": [], + "surprises": [], + "recommendation": "r", + } + ) + assert p.themes == [] + assert p.agreements == [] + + def test_real_lists_pass_through_unchanged(self): + p = PartialSummary.model_validate( + { + "summary": "s", + "themes": ["- keeps literal dash items intact"], + "agreements": ["a"], + "disagreements": ["d"], + "surprises": ["sp"], + "recommendation": "r", + } + ) + assert p.themes == ["- keeps literal dash items intact"] + + +class TestMapReduceToleratesStringListFields: + """End-to-end regression through synthesize_panel_mapreduce: a map + call whose SynthesisResult carries newline-bulleted strings for the + list fields must NOT raise MapPhaseFailure; the coerced lists are + written back onto the map result before reduce.""" + + def test_string_valued_map_partial_parses_and_reduce_runs(self): + from synth_panel import synthesis + + map_results: list[SynthesisResult] = [] + + def _fake_synth(client, panelists, questions, **kwargs): + q_text = questions[0].get("text") if isinstance(questions[0], dict) else str(questions[0]) + if kwargs.get("custom_prompt") == synthesis._REDUCE_PROMPT_TEMPLATE: + return SynthesisResult( + summary="reduced", + themes=["cross-question theme"], + agreements=["a"], + disagreements=["d"], + surprises=["s"], + recommendation="r", + ) + res = SynthesisResult( + summary=f"summary-for-{q_text}", + themes="\n- Distinctiveness and identity\n- Pricing anxiety", # type: ignore[arg-type] + agreements="- broad agreement", # type: ignore[arg-type] + disagreements="", # type: ignore[arg-type] + surprises="\n- One surprise", # type: ignore[arg-type] + recommendation="rec", + ) + map_results.append(res) + return res + + with patch.object(synthesis, "synthesize_panel", side_effect=_fake_synth): + result = synthesize_panel_mapreduce( + object(), + _PANELISTS, + _QUESTIONS, + model="sonnet", + max_workers=1, + ) + assert isinstance(result, SynthesisResult) + assert result.summary == "reduced" + # Coerced lists were written back onto each map partial. + for res in map_results: + assert res.themes == ["Distinctiveness and identity", "Pricing anxiety"] + assert res.agreements == ["broad agreement"] + assert res.disagreements == [] + assert res.surprises == ["One surprise"]