diff --git a/.github/scripts/verify-ci-checks.sh b/.github/scripts/verify-ci-checks.sh index cdb7fb92..57602484 100755 --- a/.github/scripts/verify-ci-checks.sh +++ b/.github/scripts/verify-ci-checks.sh @@ -13,7 +13,7 @@ # concurrency group cancels one of the twin runs — park a cancelled or # failed run next to the successful same-named run, and those superseded # runs never turn green. Counting all of them blocked SynthBench v0.3.1 -# (PR #334) and SynthPanel's first v1.6.0 attempt, forcing manual tags. +# (PR #334) and Althing's first v1.6.0 attempt, forcing manual tags. # Only the newest run per (app, name) reflects the commit's real status; # GitHub's own merge box collapses check runs the same way. # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b2f8775..d8ac8778 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,8 +30,8 @@ jobs: - run: ruff check . - run: ruff format --check . - name: Guard against hyphenated sibling-project regression (sb-1el9) - # The sibling project's canonical names are `synthpanel` (PyPI package) - # and `SynthPanel` (GitHub repo). The stale pre-rename hyphenated form + # The sibling project's canonical names are `althing` (PyPI package) + # and `Althing` (GitHub repo). The stale pre-rename hyphenated form # should never reappear. Historical audit notes (post-flip-audit-*.md) # are allowed to reference the old name while documenting the rename. # The pattern is assembled at runtime so this step doesn't match itself. @@ -46,7 +46,7 @@ jobs: --exclude="post-flip-audit-*.md" \ . || true) if [ -n "$matches" ]; then - echo "::error::Hyphenated sibling-project name found — use 'synthpanel' (PyPI) or 'SynthPanel' (repo)." + echo "::error::Hyphenated sibling-project name found — use 'althing' (PyPI) or 'Althing' (repo)." echo "$matches" exit 1 fi @@ -329,7 +329,7 @@ jobs: - name: Mint App token # Pushing with this token (instead of GITHUB_TOKEN) makes the # baseline commit fire `pull_request: synchronize`, so CI re-runs on - # the new HEAD automatically. Same pattern as SynthPanel's + # the new HEAD automatically. Same pattern as Althing's # auto-tag.yml release-bot push. if: steps.vrt_gate.outputs.run == 'true' id: app_token diff --git a/.github/workflows/semver-label.yml b/.github/workflows/semver-label.yml index 6a3981ec..55860fc8 100644 --- a/.github/workflows/semver-label.yml +++ b/.github/workflows/semver-label.yml @@ -4,7 +4,7 @@ name: Semver Label # (semver:patch / semver:minor / semver:major to bump, semver:skip to opt # out) BEFORE merge. auto-tag.yml reads the label post-merge to cut the # release; without this gate a missing label silently skips the release -# behind a green check (the failure mode SynthPanel hit twice — sp-42i, +# behind a green check (the failure mode Althing hit twice — sp-42i, # sy-73j — before adopting the same pre-merge check). on: diff --git a/FINDINGS.md b/FINDINGS.md index 02adfeaf..87030c2f 100644 --- a/FINDINGS.md +++ b/FINDINGS.md @@ -2,6 +2,12 @@ **Date**: 2026-04-12 (session-1 experiments) | **Numbers regenerated from artifacts** — see below +> **Naming note (2026-07):** the `synthpanel` provider referenced throughout +> these findings was renamed to **[althing](https://althing.dev)**. Result +> artifacts and generated tables retain the provider id `synthpanel` they +> were recorded under; `--provider synthpanel` remains a working alias for +> the `althing` provider. + > All quantitative tables in this document sit between > `` / `` markers and are > rendered by `scripts/generate-findings-md.py` from the per-question rows @@ -34,9 +40,9 @@ asymmetries in how LLMs represent different demographic groups. | Dataset | 3-model ensemble SPS | Best single model | Random baseline | |---------|---------------------|-------------------|-----------------| -| globalopinionqa (n=100) | **0.813** | 0.786 (SynthPanel (GPT-4o-mini), product) | 0.774 (n=100) | +| globalopinionqa (n=100) | **0.813** | 0.786 (Althing (GPT-4o-mini), product) | 0.774 (n=100) | | opinionsqa (n=684) | **0.877** | 0.829 (Gemini 2.5 Flash, raw) | 0.763 (n=684) | -| subpop (n=200) | **0.858** | 0.821 (SynthPanel (Gemini Flash Lite), product) | 0.757 (n=200) | +| subpop (n=200) | **0.858** | 0.821 (Althing (Gemini Flash Lite), product) | 0.757 (n=200) | Note on interpretation: SPS is a composite parity score, not a @@ -140,7 +146,7 @@ run vs the identical configuration with schema-forced structured capture count — the elicitation surface is the only variable. -**SynthPanel (Haiku 4.5)** (product) on gss — n=75 questions × 30 samples per arm, refusal detector v2; the arms differ only in elicitation mode: +**Althing (Haiku 4.5)** (product) on gss — n=75 questions × 30 samples per arm, refusal detector v2; the arms differ only in elicitation mode: | Metric | Natural | Structured | Δ (structured − natural) | |--------|---------|------------|--------------------------| @@ -252,9 +258,9 @@ three runs costs roughly 3× a single-model run. | Dataset | Best single model | Equal blend | Improvement | Random baseline | |---------|-------------------|-------------|-------------|-----------------| -| globalopinionqa (100q) | 0.786 (SynthPanel (GPT-4o-mini), product) | **0.813** | **+2.7 pts** | 0.774 (n=100) | +| globalopinionqa (100q) | 0.786 (Althing (GPT-4o-mini), product) | **0.813** | **+2.7 pts** | 0.774 (n=100) | | opinionsqa (684q) | 0.829 (Gemini 2.5 Flash, raw) | **0.877** | **+4.8 pts** | 0.763 (n=684) | -| subpop (200q) | 0.821 (SynthPanel (Gemini Flash Lite), product) | **0.858** | **+3.7 pts** | 0.757 (n=200) | +| subpop (200q) | 0.821 (Althing (Gemini Flash Lite), product) | **0.858** | **+3.7 pts** | 0.757 (n=200) | Comparison set: best_single = highest recomputed-SPS non-ensemble, non-baseline leaderboard row (raw or product framework) evaluated on at least as many questions as the ensemble, after per-(model, framework, dataset) dedup; random_baseline = recomputed SPS of the random-baseline run for the same dataset (n in random_baseline_n). @@ -277,25 +283,25 @@ artifacts strip `human_distribution` (#308). | Constituent pair | Pearson r (per-question JSD) | n | |------------------|------------------------------|---| -| SynthPanel (Haiku 4.5) ↔ SynthPanel (Gemini Flash Lite) | 0.419 | 100 | -| SynthPanel (Haiku 4.5) ↔ SynthPanel (GPT-4o-mini) | 0.308 | 100 | -| SynthPanel (Gemini Flash Lite) ↔ SynthPanel (GPT-4o-mini) | 0.493 | 100 | +| Althing (Haiku 4.5) ↔ Althing (Gemini Flash Lite) | 0.419 | 100 | +| Althing (Haiku 4.5) ↔ Althing (GPT-4o-mini) | 0.308 | 100 | +| Althing (Gemini Flash Lite) ↔ Althing (GPT-4o-mini) | 0.493 | 100 | **opinionsqa** (n=684 common questions, mean pairwise r = 0.295): | Constituent pair | Pearson r (per-question JSD) | n | |------------------|------------------------------|---| -| SynthPanel (Haiku 4.5) ↔ SynthPanel (Gemini Flash Lite) | 0.224 | 684 | -| SynthPanel (Haiku 4.5) ↔ SynthPanel (GPT-4o-mini) | 0.355 | 684 | -| SynthPanel (Gemini Flash Lite) ↔ SynthPanel (GPT-4o-mini) | 0.308 | 684 | +| Althing (Haiku 4.5) ↔ Althing (Gemini Flash Lite) | 0.224 | 684 | +| Althing (Haiku 4.5) ↔ Althing (GPT-4o-mini) | 0.355 | 684 | +| Althing (Gemini Flash Lite) ↔ Althing (GPT-4o-mini) | 0.308 | 684 | **subpop** (n=200 common questions, mean pairwise r = 0.322): | Constituent pair | Pearson r (per-question JSD) | n | |------------------|------------------------------|---| -| SynthPanel (Haiku 4.5) ↔ SynthPanel (Gemini Flash Lite) | 0.282 | 200 | -| SynthPanel (Haiku 4.5) ↔ SynthPanel (GPT-4o-mini) | 0.260 | 200 | -| SynthPanel (Gemini Flash Lite) ↔ SynthPanel (GPT-4o-mini) | 0.425 | 200 | +| Althing (Haiku 4.5) ↔ Althing (Gemini Flash Lite) | 0.282 | 200 | +| Althing (Haiku 4.5) ↔ Althing (GPT-4o-mini) | 0.260 | 200 | +| Althing (Gemini Flash Lite) ↔ Althing (GPT-4o-mini) | 0.425 | 200 | The constituents' errors are **moderately positively correlated** (pairwise r 0.22–0.49 on per-question JSD; 0.27–0.44 on signed per-option residuals, see asserted constants) — not uncorrelated. The ensemble gain comes from partial, not full, independence of errors; the earlier "uncorrelated errors" framing overstated it. @@ -403,8 +409,8 @@ gated files are committed without `human_distribution`). | Provider | Framework | Template | Dataset | Mean model nonresponse | Mean human nonresponse | Mean abs gap | n | |----------|-----------|----------|---------|------------------------|------------------------|--------------|---| | Gemini 2.5 Flash | raw | default | gss | 0.106 | 0.056 | **0.126** | 75 | -| SynthPanel (Haiku 4.5) | product | structured | gss | 0.064 | 0.056 | **0.095** | 75 | -| SynthPanel (Haiku 4.5) | product | default | gss | 0.067 | 0.056 | **0.094** | 75 | +| Althing (Haiku 4.5) | product | structured | gss | 0.064 | 0.056 | **0.095** | 75 | +| Althing (Haiku 4.5) | product | default | gss | 0.067 | 0.056 | **0.094** | 75 | Comparison set: Per deduped non-baseline run whose committed per-question rows still carry human_distribution (full-tier datasets; gated files are stripped per #308): mean |model explicit-nonresponse mass (DK-style option mass + parsed refusal rate) - human's|, plus the five items with the largest model over-selection. diff --git a/METHODOLOGY.md b/METHODOLOGY.md index f1538319..9faafb21 100644 --- a/METHODOLOGY.md +++ b/METHODOLOGY.md @@ -33,7 +33,7 @@ For the full technical details, read on. ### 1.1 Analysis of the Two Frameworks -**Proposed (synthpanel datascientist) — 4 sub-scores:** +**Proposed (althing datascientist) — 4 sub-scores:** | Metric | Measures | Method | |--------|----------|--------| | P_dist | Distributional parity | Jensen-Shannon divergence | @@ -144,7 +144,7 @@ default distribution. P_cond isolates the *conditioning mechanism itself*. **Conditioning protocol**: Each provider is tested with the persona spec it supports. For raw LLMs, we test all three OpinionsQA styles (QA, BIO, PORTRAY) -and take the max. For dedicated providers (synthpanel, Ditto), we use their +and take the max. For dedicated providers (althing, Ditto), we use their native persona interface. #### P_sub — Subgroup Consistency @@ -269,7 +269,7 @@ class Distribution: @dataclass class ProviderMetadata: """Provider identification and capabilities.""" - name: str # e.g. "synthpanel", "openai-gpt4o" + name: str # e.g. "althing", "openai-gpt4o" version: str # Semantic version or model identifier supports_logprobs: bool # Can return token log-probabilities supports_distribution: bool # Can return Distribution directly @@ -329,7 +329,7 @@ class SynthRespondent(Protocol): | Provider | Adapter | Distribution Method | Notes | |----------|---------|-------------------|-------| -| **synthpanel** | `SynthPanelAdapter` | Reported (native API returns distribution) | Our reference provider | +| **althing** | `AlthingAdapter` | Reported (native API returns distribution) | Our reference provider | | **OpenAI (GPT-4o, etc.)** | `OpenAIAdapter` | Logprobs (top-k token logprobs) | `conditioning_style` mapped to system/user prompt | | **Anthropic (Claude)** | `AnthropicAdapter` | Sampling (no logprobs available) | Requires n_samples >= 30 for stable distributions | | **Ditto** | `DittoAdapter` | Reported (if API returns distribution) or Sampling | Pending API access to confirm capabilities | @@ -372,7 +372,7 @@ class OpenAIAdapter: CLI invocation: ```bash synthbench run --provider openai --model gpt-4o --suite core -synthbench run --provider synthpanel --suite full +synthbench run --provider althing --suite full synthbench run --provider generic-http --config ./my-provider.yaml --suite core ``` @@ -617,9 +617,9 @@ Every SynthBench run produces a JSON artifact: "timestamp": "2026-04-10T12:00:00Z", "suite": "core", "provider": { - "name": "synthpanel", + "name": "althing", "version": "2.1.0", - "adapter": "SynthPanelAdapter", + "adapter": "AlthingAdapter", "config_hash": "sha256:..." }, "scores": { @@ -669,7 +669,7 @@ Every SynthBench run produces a JSON artifact: ================================================================ SYNTHBENCH v1.0 SCORE CARD ================================================================ - Provider: synthpanel v2.1.0 + Provider: althing v2.1.0 Suite: SynthBench-Core (300 questions) Date: 2026-04-10 Run ID: run-20260410-a1b2c3 @@ -720,7 +720,7 @@ For multi-provider comparisons: ``` | Provider | Version | SPS | P_dist | P_rank | P_cond | P_sub | P_refuse | Suite | |-----------------|---------|------|--------|--------|--------|-------|----------|-------| -| synthpanel | 2.1.0 | 0.74 | 0.81 | 0.77 | 0.68 | 0.72 | 0.71 | core | +| althing | 2.1.0 | 0.74 | 0.81 | 0.77 | 0.68 | 0.72 | 0.71 | core | | GPT-4o (portray)| 2026-03 | 0.61 | 0.67 | 0.63 | 0.55 | 0.58 | 0.62 | core | | Claude (portray)| sonnet | 0.59 | 0.65 | 0.61 | 0.52 | 0.60 | 0.58 | core | | GPT-4o (raw) | 2026-03 | 0.48 | 0.55 | 0.52 | 0.00 | 0.45 | 0.40 | core | diff --git a/README.md b/README.md index 28300945..bdd8bdc0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Open benchmark harness for synthetic survey respondent quality. **The MLPerf of synthetic UXR.** -SynthBench measures how well synthetic respondent systems (like [synthpanel](https://github.com/DataViking-Tech/SynthPanel), Ditto, Synthetic Users, or raw ChatGPT prompting) reproduce real human survey response patterns against real Pew American Trends Panel and GlobalOpinionQA ground truth — so "it sounds plausible" gets replaced with a measurable similarity score. +SynthBench measures how well synthetic respondent systems (like [althing](https://github.com/DataViking-Tech/Althing), Ditto, Synthetic Users, or raw ChatGPT prompting) reproduce real human survey response patterns against real Pew American Trends Panel and GlobalOpinionQA ground truth — so "it sounds plausible" gets replaced with a measurable similarity score. ## Quick Start @@ -203,7 +203,7 @@ The leaderboard JSON carries per-row cost fields and a top-level runs listed in `config.ensemble_sources`. - `pricing_snapshot` records the per-model `input_per_1m` / `output_per_1m` rates used for this publish run, the `snapshot_date` anchor comment, and - the installed `synth_panel_version` that produced the rates. + the installed `althing_version` that produced the rates. This lets downstream consumers audit which pricing table produced which `cost_usd` and reconcile against provider-reported billing without guessing @@ -220,7 +220,7 @@ both curves side-by-side. See [`docs/convergence-analysis.md`](docs/convergence-analysis.md) for the JSON schema, CLI flags, and the `load_convergence_baseline` integration -surface that synthpanel's `--calibrate-against DATASET:QUESTION` flag +surface that althing's `--calibrate-against DATASET:QUESTION` flag consumes. ## Citation diff --git a/crew/architect/docs/accessibility-guidelines.md b/crew/architect/docs/accessibility-guidelines.md index 6c62bd2d..7c5f259d 100644 --- a/crew/architect/docs/accessibility-guidelines.md +++ b/crew/architect/docs/accessibility-guidelines.md @@ -228,7 +228,7 @@ Color-only encoding fails for ~8% of male users (colorblindness). For small stac ### Badges - `bg-accent/15 text-accent` — acceptable with the darker accent token (verify per-instance). -- Provider tags (`bg-provider-synthpanel/10 text-provider-synthpanel`) — provider colors are hex literals; check each one against the surfaces they sit on. +- Provider tags (`bg-provider-althing/10 text-provider-althing`) — provider colors are hex literals; check each one against the surfaces they sit on. --- diff --git a/docs/ai-distribution-and-pitch.md b/docs/ai-distribution-and-pitch.md index f65fd18f..6f7204e3 100644 --- a/docs/ai-distribution-and-pitch.md +++ b/docs/ai-distribution-and-pitch.md @@ -12,18 +12,18 @@ Ranked by **reach ÷ effort for the AE Miami audience** (applied AI engineers, agent builders, AI-native startups). Effort estimates assume one engineer who knows the codebase. -Products span SynthPanel (CLI + MCP server), SynthBench (CLI + public leaderboard site), +Products span Althing (CLI + MCP server), SynthBench (CLI + public leaderboard site), and Traitprint (hosted web app with API). Different channels map to different products. ### Tier S — Ship before Tuesday (table stakes + high ROI) | # | Channel | Product | Effort | Reach | Why now | |---|---------|---------|--------|-------|---------| -| 1 | **MCP Registry (GitHub + Anthropic directories)** | SynthPanel | ~2h | Very high | Server already exists (`synthpanel mcp-serve`). Listing is a metadata PR. Not being listed on Monday reads as "not serious about MCP" to an Anthropic-heavy audience. **TABLE STAKES.** | +| 1 | **MCP Registry (GitHub + Anthropic directories)** | Althing | ~2h | Very high | Server already exists (`althing mcp-serve`). Listing is a metadata PR. Not being listed on Monday reads as "not serious about MCP" to an Anthropic-heavy audience. **TABLE STAKES.** | | 2 | **Hugging Face Space (benchmark viewer)** | SynthBench | 1 day | Very high | Every credible eval benchmark lives on HF Spaces (HELM, MMLU, HumanEval). A drive-by researcher expects to find SynthBench there. Gradio wrapper around leaderboard.json + a "run the core suite" button. **TABLE STAKES.** | | 3 | **Hugging Face Dataset (scored results)** | SynthBench | half day | High | Publish `leaderboard-results/` as a versioned HF dataset. Free citation path for researchers. Pairs with #2. | | 4 | **Papers With Code listing** | SynthBench | 2h | Mid-high | Benchmark metadata + leaderboard mirror. Table-stakes for the research-leaning slice of the audience. | -| 5 | **Claude Code skill pack** — `/synthpanel-run`, `/synthbench-submit`, `/traitprint-profile` | All three | 1 day for three skills | High | Anthropic-heavy audience runs Claude Code. A ready-made skill they can `plugin install` during the demo is a conversion engine. **Differentiator** — few vendors have shipped skills yet. | +| 5 | **Claude Code skill pack** — `/althing-run`, `/synthbench-submit`, `/traitprint-profile` | All three | 1 day for three skills | High | Anthropic-heavy audience runs Claude Code. A ready-made skill they can `plugin install` during the demo is a conversion engine. **Differentiator** — few vendors have shipped skills yet. | **Tier S bundle: ~3 engineer-days total. Ship all five before Monday EOD.** @@ -31,18 +31,18 @@ and Traitprint (hosted web app with API). Different channels map to different pr | # | Channel | Product | Effort | Reach | Notes | |---|---------|---------|--------|-------|-------| -| 6 | **Composio tool listing** | SynthPanel | 1-2 days | Mid-high | Composio aggregates tools for LangChain/CrewAI/Autogen. One listing unlocks multiple frameworks. | -| 7 | **LangChain tool package** (`langchain-synthpanel`) | SynthPanel | 1 day | High | Thin wrapper over MCP tools. Publish to PyPI + LangChain Hub. | -| 8 | **CrewAI tool package** (`crewai-synthpanel`) | SynthPanel | 1 day | Mid | Same wrapper shape; separate import path. | -| 9 | **Cursor / Windsurf / Zed MCP visibility** | SynthPanel | 0 incremental | High | All three consume MCP; once #1 ships, this is free. Add one paragraph + screenshot to README. | -| 10 | **ChatGPT Custom GPT — "SynthPanel Focus Group"** | SynthPanel | 2-4h + hosted action endpoint | Mid (decaying) | GPT Store discoverability has softened, but the ChatGPT install base is still massive. Low-lift hedge. | +| 6 | **Composio tool listing** | Althing | 1-2 days | Mid-high | Composio aggregates tools for LangChain/CrewAI/Autogen. One listing unlocks multiple frameworks. | +| 7 | **LangChain tool package** (`langchain-althing`) | Althing | 1 day | High | Thin wrapper over MCP tools. Publish to PyPI + LangChain Hub. | +| 8 | **CrewAI tool package** (`crewai-althing`) | Althing | 1 day | Mid | Same wrapper shape; separate import path. | +| 9 | **Cursor / Windsurf / Zed MCP visibility** | Althing | 0 incremental | High | All three consume MCP; once #1 ships, this is free. Add one paragraph + screenshot to README. | +| 10 | **ChatGPT Custom GPT — "Althing Focus Group"** | Althing | 2-4h + hosted action endpoint | Mid (decaying) | GPT Store discoverability has softened, but the ChatGPT install base is still massive. Low-lift hedge. | ### Tier B — Ship within 1 quarter (steady background investment) | # | Channel | Product | Effort | Reach | Notes | |---|---------|---------|--------|-------|-------| -| 11 | **n8n community node** | SynthPanel | 2 days | Mid | No-code automation crowd. Template: "Slack thread → SynthPanel focus group → Notion doc". | -| 12 | **Zapier / Make connectors** | SynthPanel / Traitprint | 3-5 days each | Mid | Higher reach than n8n for non-technical buyers. Traitprint is the stronger fit here (HR/recruiting workflows). | +| 11 | **n8n community node** | Althing | 2 days | Mid | No-code automation crowd. Template: "Slack thread → Althing focus group → Notion doc". | +| 12 | **Zapier / Make connectors** | Althing / Traitprint | 3-5 days each | Mid | Higher reach than n8n for non-technical buyers. Traitprint is the stronger fit here (HR/recruiting workflows). | | 13 | **Hugging Face Space for Traitprint demo** | Traitprint | 1 day | Mid | Low stakes — primarily an SEO/discovery play. | | 14 | **dstack / Replicate** | None | N/A | N/A | Poor fit. These host inference models, not eval pipelines or web apps. Skip. | | 15 | **VS Code extension** | None | N/A | Low | Nothing in the portfolio is an editor-time tool. Skip. | @@ -52,17 +52,17 @@ and Traitprint (hosted web app with API). Different channels map to different pr | # | Channel | Product | Effort | Reach | Notes | |---|---------|---------|--------|-------|-------| -| 17 | **AWS Bedrock Agents / Azure AI Agent Service** | SynthPanel | 1-2 weeks each | Mid (enterprise) | Only if an enterprise design partner materializes. | +| 17 | **AWS Bedrock Agents / Azure AI Agent Service** | Althing | 1-2 weeks each | Mid (enterprise) | Only if an enterprise design partner materializes. | | 18 | **Salesforce AgentForce / MuleSoft Anypoint** | Traitprint | 2-3 weeks | Mid (HR tech) | Only if HR-tech is a target vertical. | -| 19 | **ElevenLabs / Deepgram agent marketplaces** | SynthPanel | Unknown | Low-mid | Voice-agent angle — real only if SynthPanel ships a voice persona mode. | +| 19 | **ElevenLabs / Deepgram agent marketplaces** | Althing | Unknown | Low-mid | Voice-agent angle — real only if Althing ships a voice persona mode. | | 20 | **arXiv paper + NeurIPS/ICML workshop submission** | SynthBench | 2-4 weeks + review cycle | Enormous long-term | Wrong time horizon for Tuesday, right time horizon for a Q3 credibility play. | ### Recommended ship order for the week **Monday (EOD goal — 4 of 5 Tier S shipped):** -1. MCP Registry listing for SynthPanel (2h) +1. MCP Registry listing for Althing (2h) 2. Papers With Code listing for SynthBench (2h) -3. Claude Code skill pack — `/synthpanel-run` + `/synthbench-submit` (1 day) +3. Claude Code skill pack — `/althing-run` + `/synthbench-submit` (1 day) 4. Hugging Face Space — SynthBench leaderboard viewer (1 day, parallelizable) **Tuesday (demo day):** Tier S #3 (HF Dataset) if capacity. Demo-safe. @@ -70,7 +70,7 @@ and Traitprint (hosted web app with API). Different channels map to different pr **Week 2-3:** Tier A (Composio + LangChain + CrewAI packages). **Demo narrative unlock:** Tier S #1 and #5 let the demo include *"install the -SynthPanel MCP in Claude Code right now — one command, try it with me"* — a +Althing MCP in Claude Code right now — one command, try it with me"* — a conversion-maximizing live interaction this audience will remember. ### What NOT to chase @@ -78,7 +78,7 @@ conversion-maximizing live interaction this audience will remember. - **ChatGPT Plugins (legacy)** — deprecated path. Custom GPTs only. - **Slack / Discord bots** — low ROI for this audience; they'll come to you. - **Fine-tuning-platform partnerships** — wrong layer; our products don't need fine-tuning. -- **NPM packages** — no JS surface worth exposing today. Skip until there's a SynthPanel-node client worth publishing. +- **NPM packages** — no JS surface worth exposing today. Skip until there's a Althing-node client worth publishing. --- @@ -114,7 +114,7 @@ and that is the pitch. > "DataViking builds the trust layer for any AI product that makes claims about > people. **Traitprint** turns a résumé into a rigorous professional profile with -> skill-level evidence. **SynthPanel** runs synthetic focus groups you can +> skill-level evidence. **Althing** runs synthetic focus groups you can > actually ship product decisions on. **SynthBench** measures, against real human > survey data, whether those synthetic people match reality — or whether your > model is just giving you trendslop. Represent real people, simulate @@ -133,7 +133,7 @@ posture. > "Every AI product now makes assumptions about humans — what they'd choose, > what they'd think, what they're qualified for. **DataViking makes those > assumptions measurable.** **Traitprint** quantifies professional identity into -> structured skill evidence. **SynthPanel** generates synthetic user populations +> structured skill evidence. **Althing** generates synthetic user populations > for research, product, and hiring decisions. **SynthBench** scores, against > real population data, how well those populations match the humans they claim > to represent. We built the ground truth so your AI doesn't have to guess. @@ -154,7 +154,7 @@ terms ("your AI doesn't have to guess"). #### Pitch C — "Synthetic UXR that leadership will sign off on" (practitioner angle) -> "DataViking turns 'what would users think?' from a hunch into data. **SynthPanel** +> "DataViking turns 'what would users think?' from a hunch into data. **Althing** > lets your team run a 500-person focus group in ten minutes. **SynthBench** > proves — with real Pew-panel data — that those synthetic respondents are > actually representative, or flags them when they aren't. **Traitprint** does @@ -198,7 +198,7 @@ ship on vibes" slightly polarizing for conservative enterprise buyers. ### One-line reduction (for bio / LinkedIn / booth signage) > **DataViking Technologies — tools for the measurable human. -> Traitprint. SynthPanel. SynthBench.** +> Traitprint. Althing. SynthBench.** ### Founder one-liner (for Wesley to use in 1:1 intros) @@ -209,7 +209,7 @@ ship on vibes" slightly polarizing for conservative enterprise buyers. ### Rejected framings (and why) - *"We're building the open evaluation stack for human-AI interaction."* — too - academic, and SynthPanel is a product not an eval. + academic, and Althing is a product not an eval. - *"DataViking is the Vercel of synthetic research."* — over-analogized, borrows credibility it shouldn't need. - *"We make LLMs honest."* — correct spirit but LLM-providers will read it as @@ -222,7 +222,7 @@ ship on vibes" slightly polarizing for conservative enterprise buyers. - [x] Readiness assessment (`ai-engineer-readiness.md`, shipped) - [x] AI distribution ranking (this doc, Part 1) - [x] Elevator pitch candidates + recommendation (this doc, Part 2) -- [ ] Pending: re-run `synthpanel prompt` with API key and append raw responses +- [ ] Pending: re-run `althing prompt` with API key and append raw responses to the readiness doc (blocker: no provider key in crew/cpo env) *End.* diff --git a/docs/ai-engineer-readiness.md b/docs/ai-engineer-readiness.md index d56c96dc..e0dc1e04 100644 --- a/docs/ai-engineer-readiness.md +++ b/docs/ai-engineer-readiness.md @@ -75,7 +75,7 @@ one-sentence "what is SPS, what does ⚠ mean" orienting callout above the table `/leaderboard`: *"✓ verified = public/private SPS within 0.05 tolerance. ⚠ flagged = divergence requires review — the row is still displayed for transparency."* Without this, a skeptical visitor sees ⚠ on row #2 (Gemini 2.5 Flash) and ⚠ on the - SynthPanel Ensemble and walks away questioning the whole board. + Althing Ensemble and walks away questioning the whole board. 4. **Missing 2026 frontier models.** The raw-LLM column is dominated by 2024-vintage models (GPT-4o-mini, Gemini 2.5 Flash, Llama 3.3 70B, Haiku 4.5, Sonnet 4). At AI @@ -129,10 +129,10 @@ one-sentence "what is SPS, what does ⚠ mean" orienting callout above the table ## 4. Synthetic survey — AI engineer attendees -### Execution note — synthpanel could not run in this session +### Execution note — althing could not run in this session -`synthpanel prompt ...` failed with `Missing API key: set ANTHROPIC_API_KEY` in my -sandbox. The CLI is installed (`/opt/homebrew/bin/synthpanel`) but no provider key +`althing prompt ...` failed with `Missing API key: set ANTHROPIC_API_KEY` in my +sandbox. The CLI is installed (`/opt/homebrew/bin/althing`) but no provider key is present in the crew/cpo environment and I am not authorized to source one. I flagged this so that a follow-up session with a key can replay the exact prompt verbatim and append raw responses to this document. @@ -140,7 +140,7 @@ verbatim and append raw responses to this document. **Command to re-run when a key is available:** ```bash -synthpanel --output-format json prompt "You are an AI engineer attending \ +althing --output-format json prompt "You are an AI engineer attending \ AI Engineer Miami 2026. You just visited synthbench.org — an open MIT-licensed \ benchmark measuring how well LLMs replicate human survey response distributions. \ It has 395 leaderboard entries, OpinionsQA + GlobalOpinionQA + SubPOP as ground \ @@ -150,12 +150,12 @@ public repo. Give an honest first impression in 3-5 sentences. What would make \ you want to submit your own benchmark run?" ``` -(Better: `synthpanel panel run` with a 10-persona "AI-engineer-attendee" pack.) +(Better: `althing panel run` with a 10-persona "AI-engineer-attendee" pack.) ### Reasoned audience simulation (substitute — label as such in public use) Based on the canonical AI-engineer-audience response patterns documented in prior -SynthPanel runs and community sentiment analysis, the expected distribution of +Althing runs and community sentiment analysis, the expected distribution of first-impression responses clusters into four archetypes: **Archetype A — "The skeptical evaluator" (~35%).** *"First question: is this @@ -186,7 +186,7 @@ That's actually reassuring. Still want to see more recent models."* - The 10-minute API-key upload path (beats PR-and-fork 3:1 for conversion) - Seeing their preferred provider/framework already represented - A clear "your row will appear in < 1 merge cycle" expectation -- The ability to add a product/framework row (SynthPanel-style) alongside a raw-LLM +- The ability to add a product/framework row (Althing-style) alongside a raw-LLM row — i.e. compete on *methodology* not just model choice **Common drop-off triggers** (what kills conversion): @@ -195,7 +195,7 @@ That's actually reassuring. Still want to see more recent models."* - Fork-and-PR friction for non-GitHub-native users - Unclear how long a run takes / what it costs -I recommend treating this simulation as directional until a real SynthPanel run +I recommend treating this simulation as directional until a real Althing run replaces it — specifically, panel A/B/C/D proportions are my estimates from conference audience composition, not measured. Conversion-trigger list is grounded in the observed UX of the current submit flow. @@ -286,8 +286,8 @@ during prep; bring up the offline copy and narrate over it. Don't improvise. - [ ] Confirm Supabase sign-in works on conference wifi for a gated dataset drill-down. - [ ] Pre-cache static site offline on the demo laptop as wifi backup. - [ ] Rehearse the 3-minute script twice end-to-end with a timer. -- [ ] When an API key is available, run the synthpanel command in §4 and append raw - responses under a "Live synthpanel output" heading below. +- [ ] When an API key is available, run the althing command in §4 and append raw + responses under a "Live althing output" heading below. --- diff --git a/docs/configs.md b/docs/configs.md index a3ea3c22..3a9b38b2 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -27,7 +27,7 @@ synthbench validate-config configs/healthcare-northcentral.yaml # row instead of bare {vendor}. synthbench submit \ --config configs/healthcare-northcentral.yaml \ - results/synthpanel_healthcare_run.json + results/althing_healthcare_run.json ``` ## Schema @@ -35,7 +35,7 @@ synthbench submit \ ```yaml # Required. Vendor adapter the result was produced against. Must match a # registered vendor; the leaderboard groups rows by this field. -vendor: synthpanel +vendor: althing # Required. Free-form version string the contributor declares. Reported on # the leaderboard row so two configs against different vendor releases stay @@ -89,7 +89,7 @@ decoding: # Optional. Free-form note shown under the row. Don't paste secrets here — # everything in the artifact is public on the leaderboard. notes: | - Optimized for healthcare workers in NC/MN/WI/IA. Beats synthpanel/default + Optimized for healthcare workers in NC/MN/WI/IA. Beats althing/default by ~6 pp on the targeted slice; tie elsewhere. ``` @@ -142,7 +142,7 @@ under their vendor default. "aggregate": { "..." }, "per_question": [ "..." ], "user_config": { - "vendor": "synthpanel", + "vendor": "althing", "vendor_version": "1.4.0", "config_name": "wesley-healthcare-northcentral", "contributor": "@wesley", diff --git a/docs/convergence-analysis.md b/docs/convergence-analysis.md index 975ab070..12feaecf 100644 --- a/docs/convergence-analysis.md +++ b/docs/convergence-analysis.md @@ -42,7 +42,7 @@ Use it for three things: 1. **Lower-bound reference.** Overlay real-sample convergence against the bootstrap curve to see how close real samples get to the i.i.d. floor. 2. **Budget planning.** If the bootstrap `convergence_n` is 2000 for a - question, no amount of synthpanel traffic below 2000 will approximate the + question, no amount of althing traffic below 2000 will approximate the true distribution — and that is before accounting for heterogeneity. 3. **Question triage.** Questions with pathologically large `convergence_n` (or no convergence on the default grid) are expensive to evaluate and @@ -166,15 +166,15 @@ direct visualization of the gap between the i.i.d. floor and real sampling. See `docs/microdata-ingestion.md` for per-dataset setup (starting with GSS; WVS and Eurobarometer follow). -## Integration: synthpanel `--calibrate-against` +## Integration: althing `--calibrate-against` -synthpanel (>= 0.9) ships a `--calibrate-against DATASET:QUESTION` flag that +althing (>= 0.9) ships a `--calibrate-against DATASET:QUESTION` flag that attaches a `per_question[key].calibration` sub-object to each convergence -payload. When that flag is set, synthpanel calls into synthbench as a soft +payload. When that flag is set, althing calls into synthbench as a soft dependency to resolve the aggregate `human_distribution` used as the calibration baseline. The call site is -`synth_panel.convergence.load_synthbench_baseline(spec)` (see -`src/synth_panel/convergence.py` in the synthpanel repo). +`althing.convergence.load_synthbench_baseline(spec)` (see +`src/althing/convergence.py` in the althing repo). The loader probes for the following attributes in order and uses the first one it finds: @@ -203,20 +203,20 @@ shaped like: } ``` -`human_distribution` is the only load-bearing field — synthpanel attaches it +`human_distribution` is the only load-bearing field — althing attaches it to every matching question's `calibration` sub-object and computes JSD -against the model's cumulative distribution using synthpanel's own local JSD +against the model's cumulative distribution using althing's own local JSD (deliberately not synthbench's, per the sp-inline-calibration spec). -`dataset` and `question_key` are filled in by synthpanel if missing. The +`dataset` and `question_key` are filled in by althing if missing. The redistribution / license / citation fields are informational and carried -through into the synthpanel run metadata. +through into the althing run metadata. `question_text` and `options` are **additive, policy-gated** fields (added in `sb-qtext`): * `question_text` — the verbatim survey-item wording. * `options` — the ordered answer labels. These are aligned with the keys of - `human_distribution` (same strings), so synthpanel can present the real + `human_distribution` (same strings), so althing can present the real answer choices to its panel and derive a `pick_one` schema whose labels match ground truth. @@ -244,7 +244,7 @@ importable from `synthbench.convergence`) as of `sb-ham8`. Tier handling: rights for them. Both error classes subclass `LookupError`; `BaselineGatedError` is itself -a subclass of `BaselineUnavailable`, so synthpanel can catch the parent +a subclass of `BaselineUnavailable`, so althing can catch the parent when the distinction does not matter. ## Cross-reference diff --git a/docs/leaderboard-json-api.md b/docs/leaderboard-json-api.md index 27dfe656..b540e35c 100644 --- a/docs/leaderboard-json-api.md +++ b/docs/leaderboard-json-api.md @@ -1,6 +1,6 @@ # Leaderboard JSON API -Stable HTTP endpoint for downstream consumers (SynthPanel `--best-model-for`, +Stable HTTP endpoint for downstream consumers (Althing `--best-model-for`, agents, dashboards) that want SynthBench's current model recommendation data without scraping the site or cloning the repo. @@ -29,9 +29,9 @@ consumers can fetch directly. "entries": [ { "rank": 1, - "config_id": "synthpanel--claude-sonnet-4--tdefault--tplcurrent--d1dd307b", - "provider": "SynthPanel (Sonnet 4)", - "model": "SynthPanel (Sonnet 4)", + "config_id": "althing--claude-sonnet-4--tdefault--tplcurrent--d1dd307b", + "provider": "Althing (Sonnet 4)", + "model": "Althing (Sonnet 4)", "dataset": "globalopinionqa", "framework": "product", "sps": 0.7966, @@ -74,7 +74,7 @@ Always present on every row: | `provider` | string | Display name of the system under test. | | `model` | string | Concrete model identity (often equal to `provider`). | | `dataset` | string | One of the `datasets` array above. | -| `framework` | string | `product`, `raw`, `synthpanel`, ... | +| `framework` | string | `product`, `raw`, `althing`, ... | | `sps` | float | Synthetic Population Similarity score, [0, 1]. | | `p_dist` / `p_rank` / `p_refuse` | float | SPS sub-metrics. | | `jsd` | float | Jensen–Shannon divergence to human distribution. | @@ -186,7 +186,7 @@ Data honesty notes: - **Patch** bump → data refresh, no schema change. - The exact URL `https://synthbench.org/data/leaderboard.json` is part of the contract. It will not move or be renamed — only its payload schema is - versioned. SynthPanel's `sy-nkh` fallback depends on this stability. + versioned. Althing's `sy-nkh` fallback depends on this stability. - **Forward compatibility**: Consumers MUST tolerate unknown top-level keys and unknown `entries[].*` fields. We add fields freely at any minor. - **Backward compatibility**: Once a field is documented as required, it stays @@ -209,7 +209,7 @@ HTTP `Date` or `Last-Modified`. ## Consumer guidance -### SynthPanel +### Althing ```python import urllib.request, json diff --git a/docs/microdata-ingestion.md b/docs/microdata-ingestion.md index 6bc2e079..2572fe88 100644 --- a/docs/microdata-ingestion.md +++ b/docs/microdata-ingestion.md @@ -16,7 +16,7 @@ have heavier tails than `1/√n` predicts). Real-sampling convergence sub-samples *actual respondents* without replacement and measures `JSD(subsample, full_population)`. Plotting both -curves on the same axes is the headline claim: *"synthpanel convergence at +curves on the same axes is the headline claim: *"althing convergence at n=Y matches real human sampling convergence at n=Z."* ## Common types diff --git a/docs/submit.md b/docs/submit.md index 0a97f199..4c0fcf7d 100644 --- a/docs/submit.md +++ b/docs/submit.md @@ -5,7 +5,7 @@ > pipeline that turns an adapter into a leaderboard-ready submission is > tracked in follow-up issues. -This is the path SynthPanel competitors take to land on the SynthBench +This is the path Althing competitors take to land on the SynthBench leaderboard without filing a bespoke evaluation pipeline. The three steps are: write an adapter, run `synthbench submit-adapter`, open a PR. diff --git a/docs/traitprint-local-signoff.md b/docs/traitprint-local-signoff.md index d5e39d49..e1f92223 100644 --- a/docs/traitprint-local-signoff.md +++ b/docs/traitprint-local-signoff.md @@ -66,7 +66,7 @@ Rationale: If that returns "did you mean traitprint-local?" you've lost them. 2. **One brand, one URL, one CLI surface.** `traitprint init` and `traitprint push` in the same binary is the cleanest mental model. -3. **Precedent in the portfolio.** SynthPanel is a single package with +3. **Precedent in the portfolio.** Althing is a single package with MCP server + CLI + library built in. Consistency. 4. **License clarity.** Entire package MIT — the cloud *service* is the paid product, the client is free. Same model as `openai` SDK @@ -224,9 +224,9 @@ weeks, credibility intact." the cloud when you want to be findable."* That line alone probably converts 10-20 signups from the room. - **Distribution leverage:** the MCP Registry listing now has a - SECOND DataViking entry (SynthPanel + Traitprint). Two tools in the + SECOND DataViking entry (Althing + Traitprint). Two tools in the registry reads as an ecosystem, not a single product. -- **Brand coherence:** SynthBench is open MIT, SynthPanel is open MIT +- **Brand coherence:** SynthBench is open MIT, Althing is open MIT with an MCP server, Traitprint-Local is open MIT with an MCP server. The "DataViking ships the open layer" story writes itself. - **Recruiter-side credibility:** enterprises evaluating Traitprint for diff --git a/ensemble_blend.py b/ensemble_blend.py index a3175e2b..17040941 100644 --- a/ensemble_blend.py +++ b/ensemble_blend.py @@ -77,22 +77,19 @@ def sps_from_components( "Haiku": { "opinionsqa": lambda f: ( "claude-haiku" in f - and "synthpanel" in f + and "althing" in f and " t=0.85" in f and " tpl=" not in f ), }, "Gemini": { "opinionsqa": lambda f: ( - "gemini" in f and "synthpanel" in f and " t=2.0" in f and " tpl=" not in f + "gemini" in f and "althing" in f and " t=2.0" in f and " tpl=" not in f ), }, "GPT-4o-mini": { "opinionsqa": lambda f: ( - "gpt-4o-mini" in f - and "synthpanel" in f - and " t=1.0" in f - and " tpl=" not in f + "gpt-4o-mini" in f and "althing" in f and " t=1.0" in f and " tpl=" not in f ), }, } @@ -100,16 +97,13 @@ def sps_from_components( # --- Default-temperature file patterns (no t= in filename) --- DEFAULT_TEMP_PATTERNS = { "Haiku": lambda f: ( - "claude-haiku" in f - and "synthpanel" in f - and " t=" not in f - and " tpl=" not in f + "claude-haiku" in f and "althing" in f and " t=" not in f and " tpl=" not in f ), "Gemini": lambda f: ( - "gemini" in f and "synthpanel" in f and " t=" not in f and " tpl=" not in f + "gemini" in f and "althing" in f and " t=" not in f and " tpl=" not in f ), "GPT-4o-mini": lambda f: ( - "gpt-4o-mini" in f and "synthpanel" in f and " t=" not in f and " tpl=" not in f + "gpt-4o-mini" in f and "althing" in f and " t=" not in f and " tpl=" not in f ), } @@ -512,7 +506,7 @@ def run_optimal_temp_experiment(): # Try anyway with the OpinionsQA matcher as fallback found = False for f in RESULTS_DIR.glob(f"{ds}_*.json"): - if " t=" in f.name and "synthpanel" in f.name: + if " t=" in f.name and "althing" in f.name: if ( ( mn == "Haiku" diff --git a/notebooks/synthpanel_tuning.ipynb b/notebooks/synthpanel_tuning.ipynb index 8b1920a3..efc71207 100644 --- a/notebooks/synthpanel_tuning.ipynb +++ b/notebooks/synthpanel_tuning.ipynb @@ -4,12 +4,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# SynthPanel Tuning with SynthBench\n", + "# Althing Tuning with SynthBench\n", "\n", "This notebook shows how to use SynthBench to score and tune\n", - "[SynthPanel](https://github.com/DataViking-Tech/synthpanel) outputs.\n", + "[Althing](https://github.com/DataViking-Tech/althing) outputs.\n", "\n", - "SynthPanel generates synthetic survey responses from persona-conditioned LLMs.\n", + "Althing generates synthetic survey responses from persona-conditioned LLMs.\n", "SynthBench measures how closely those responses match real human data.\n", "Together, they form a generate-then-score loop for tuning synthetic survey quality.\n", "\n", @@ -29,7 +29,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install synthbench synthpanel" + "!pip install synthbench althing" ] }, { @@ -85,7 +85,7 @@ "source": [ "## 3. Create Personas\n", "\n", - "Personas define the demographic profiles that SynthPanel will role-play.\n", + "Personas define the demographic profiles that Althing will role-play.\n", "Different personas should produce different answer distributions — that's\n", "the whole point of synthetic panels." ] @@ -129,13 +129,13 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Run SynthPanel\n", + "## 4. Run Althing\n", "\n", "Generate synthetic responses using your instrument and personas.\n", "This calls the LLM API — make sure your API key is set.\n", "\n", - "**Note**: If synthpanel's CLI interface differs from what's shown here,\n", - "check `synthpanel --help` for the current syntax." + "**Note**: If althing's CLI interface differs from what's shown here,\n", + "check `althing --help` for the current syntax." ] }, { @@ -150,13 +150,13 @@ "# os.environ[\"OPENROUTER_API_KEY\"] = \"sk-or-...\"\n", "\n", "if os.environ.get(\"OPENROUTER_API_KEY\"):\n", - " !synthpanel run \\\n", + " !althing run \\\n", " --instrument mini_instrument.yaml \\\n", " --personas mini_personas.yaml \\\n", " --output ./sp_output\n", - " print(\"\\nSynthPanel output saved to ./sp_output/\")\n", + " print(\"\\nAlthing output saved to ./sp_output/\")\n", "else:\n", - " print(\"Set OPENROUTER_API_KEY to run SynthPanel.\")\n", + " print(\"Set OPENROUTER_API_KEY to run Althing.\")\n", " print(\"Remaining cells show what the workflow looks like.\")" ] }, @@ -166,7 +166,7 @@ "source": [ "## 5. Score the Output with SynthBench\n", "\n", - "Now we use SynthBench's synthpanel provider adapter to score the output.\n", + "Now we use SynthBench's althing provider adapter to score the output.\n", "This compares the synthetic responses against real human ground truth data." ] }, @@ -179,14 +179,14 @@ "sp_output = Path(\"./sp_output\")\n", "\n", "if sp_output.exists() and list(sp_output.glob(\"*.json\")):\n", - " # Score using the synthpanel adapter\n", + " # Score using the althing adapter\n", " !synthbench run \\\n", - " --provider synthpanel \\\n", + " --provider althing \\\n", " --suite smoke \\\n", " --samples 10 \\\n", " --output ./sb_scores\n", "else:\n", - " print(\"No SynthPanel output found. Run cell 4 first.\")" + " print(\"No Althing output found. Run cell 4 first.\")" ] }, { @@ -199,7 +199,7 @@ "Lower temperature = more consistent but potentially less diverse responses.\n", "Higher temperature = more varied but potentially noisier.\n", "\n", - "Run SynthPanel at different temperatures and compare the SynthBench scores\n", + "Run Althing at different temperatures and compare the SynthBench scores\n", "to find the sweet spot." ] }, @@ -210,20 +210,20 @@ "outputs": [], "source": [ "# This cell demonstrates the tuning loop concept.\n", - "# When synthpanel supports --temperature, you can run:\n", + "# When althing supports --temperature, you can run:\n", "#\n", "# for temp in [0.3, 0.7, 1.0]:\n", - "# !synthpanel run --instrument mini_instrument.yaml \\\n", + "# !althing run --instrument mini_instrument.yaml \\\n", "# --personas mini_personas.yaml \\\n", "# --temperature {temp} \\\n", "# --output ./sp_temp_{temp}\n", - "# !synthbench run --provider synthpanel \\\n", + "# !synthbench run --provider althing \\\n", "# --suite smoke --output ./sb_temp_{temp}\n", "#\n", "# Then compare the scores to find the optimal temperature.\n", "\n", "print(\"Temperature tuning workflow:\")\n", - "print(\" 1. Run synthpanel at temp=0.3, 0.7, 1.0\")\n", + "print(\" 1. Run althing at temp=0.3, 0.7, 1.0\")\n", "print(\" 2. Score each output with synthbench\")\n", "print(\" 3. Compare SPS scores to find the sweet spot\")\n", "print()\n", diff --git a/pyproject.toml b/pyproject.toml index 320bd766..eafe9a0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ keywords = [ "globalopinionqa", "gss", "convergence", - "synthpanel", + "althing", ] dependencies = [ "click>=8.0", @@ -31,7 +31,7 @@ dependencies = [ "numpy>=1.24", "pyyaml>=6.0", "scipy>=1.10", - "synthpanel>=0.2.0", + "althing>=1.7.0", ] [project.optional-dependencies] diff --git a/scripts/backfill-leaderboard-metrics.py b/scripts/backfill-leaderboard-metrics.py index 4ad6ef36..8f5d2176 100755 --- a/scripts/backfill-leaderboard-metrics.py +++ b/scripts/backfill-leaderboard-metrics.py @@ -3,7 +3,7 @@ Two classes of historical drift this resolves (sb-7gn): -1. DIST_SUM — synthpanel runs published before sb-7bg (PR #138) divided +1. DIST_SUM — althing runs published before sb-7bg (PR #138) divided per-question option counts by total samples (incl. refusals/parse-failures), so model_distribution sums to <1.0 (typically 0.83 or 0.91). The new runner renormalizes over valid options before publishing. We bring diff --git a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/home-dark.png b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/home-dark.png index 5d39b746..ce06f00a 100644 Binary files a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/home-dark.png and b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/home-dark.png differ diff --git a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/leaderboard-dark.png b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/leaderboard-dark.png index 40cbcac2..6e0915a0 100644 Binary files a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/leaderboard-dark.png and b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/leaderboard-dark.png differ diff --git a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/run-detail-dark.png b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/run-detail-dark.png index e7f6eb9f..822e6034 100644 Binary files a/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/run-detail-dark.png and b/site/e2e/visual/__screenshots__/smoke.visual.spec.ts/desktop-chromium/run-detail-dark.png differ diff --git a/site/public/_headers b/site/public/_headers index 65b727da..6b804a20 100644 --- a/site/public/_headers +++ b/site/public/_headers @@ -4,7 +4,7 @@ # Rules are applied in source order; later rules can override earlier ones. # Public JSON data endpoints — long-ish browser cache + longer edge cache, with -# stale-while-revalidate so consumers (SynthPanel, agents, dashboards) never +# stale-while-revalidate so consumers (Althing, agents, dashboards) never # block on a slow upstream regeneration. The endpoint itself sets the same # Cache-Control in src/pages/data/leaderboard.json.ts; this rule wins on CF # Pages because static-asset upload strips runtime headers from the file. diff --git a/site/src/components/findings/ConvergenceLine.astro b/site/src/components/findings/ConvergenceLine.astro index e0cc6243..e153c7c0 100644 --- a/site/src/components/findings/ConvergenceLine.astro +++ b/site/src/components/findings/ConvergenceLine.astro @@ -11,7 +11,7 @@ const convergence = leaderboard.convergence ?? []; // Map raw provider strings to clean display names. Only providers that map to // the target display names below are kept on the chart. const providerDisplay: Record = { - "ensemble/3-model-blend": "SynthPanel Ensemble (3-model)", + "ensemble/3-model-blend": "Althing Ensemble (3-model)", "openrouter/anthropic/claude-haiku-4-5": "Claude Haiku 4.5", "raw-anthropic/claude-haiku-4-5-20251001": "Claude Haiku 4.5", "openrouter/openai/gpt-4o-mini": "GPT-4o-mini", @@ -20,14 +20,14 @@ const providerDisplay: Record = { }; const targetDisplayOrder = [ - "SynthPanel Ensemble (3-model)", + "Althing Ensemble (3-model)", "Claude Haiku 4.5", "Gemini Flash Lite", "GPT-4o-mini", ]; const modelColors: Record = { - "SynthPanel Ensemble (3-model)": "var(--color-provider-synthpanel)", + "Althing Ensemble (3-model)": "var(--color-provider-althing)", "Claude Haiku 4.5": "#6366f1", "Gemini Flash Lite": "#0891b2", "GPT-4o-mini": "#d97706", diff --git a/site/src/components/findings/EnsembleGain.astro b/site/src/components/findings/EnsembleGain.astro index 45a35e87..099dae76 100644 --- a/site/src/components/findings/EnsembleGain.astro +++ b/site/src/components/findings/EnsembleGain.astro @@ -67,7 +67,7 @@ const chartOption = { name: "Ensemble", type: "bar" as const, data: ensembleSpsData, - itemStyle: { color: "var(--color-provider-synthpanel)" }, + itemStyle: { color: "var(--color-provider-althing)" }, barMaxWidth: 50, emphasis: { itemStyle: { shadowBlur: 10, shadowColor: "rgba(0,0,0,0.3)" }, @@ -100,7 +100,7 @@ const allR = errorCorrelation.flatMap((d) => d.pairs.map((p) => p.pearson_r)); const rRange = allR.length > 0 ? `${Math.min(...allR).toFixed(2)}–${Math.max(...allR).toFixed(2)}` : ""; -// Compact model label: "SynthPanel (Haiku 4.5)" -> "Haiku 4.5". +// Compact model label: "Althing (Haiku 4.5)" -> "Haiku 4.5". function shortName(name: string): string { const m = name.match(/\(([^)]+)\)/); return m ? m[1] : name; diff --git a/site/src/components/findings/EnsembleMethodology.astro b/site/src/components/findings/EnsembleMethodology.astro index b186b683..5a79c4d5 100644 --- a/site/src/components/findings/EnsembleMethodology.astro +++ b/site/src/components/findings/EnsembleMethodology.astro @@ -2,7 +2,7 @@ const cards = [ { heading: "What we blend", - body: "Per-question response distributions from three models — Claude Haiku 4.5, Gemini Flash Lite, and GPT-4o-mini — run through the SynthPanel conditioning framework on the same inputs.", + body: "Per-question response distributions from three models — Claude Haiku 4.5, Gemini Flash Lite, and GPT-4o-mini — run through the Althing conditioning framework on the same inputs.", }, { heading: "How the blend works", diff --git a/site/src/components/findings/TopicGroupedBar.astro b/site/src/components/findings/TopicGroupedBar.astro index f7fb0d18..be1f9cad 100644 --- a/site/src/components/findings/TopicGroupedBar.astro +++ b/site/src/components/findings/TopicGroupedBar.astro @@ -46,11 +46,11 @@ for (const entry of bestByKey.values()) { const models = [...bestByModel.values()].sort((a, b) => b.sps - a.sps); const modelColors: Record = { - "SynthPanel Ensemble (3-model)": "var(--color-provider-synthpanel)", - "SynthPanel (Haiku 4.5)": "#7c3aed", - "SynthPanel (Gemini Flash Lite)": "#0891b2", - "SynthPanel (GPT-4o-mini)": "#d97706", - "SynthPanel (Sonnet 4)": "#9333ea", + "Althing Ensemble (3-model)": "var(--color-provider-althing)", + "Althing (Haiku 4.5)": "#7c3aed", + "Althing (Gemini Flash Lite)": "#0891b2", + "Althing (GPT-4o-mini)": "#d97706", + "Althing (Sonnet 4)": "#9333ea", "Claude Haiku 4.5": "#6366f1", "Gemini Flash Lite (direct)": "#22d3ee", "GPT-4o-mini": "#f59e0b", diff --git a/site/src/components/home/CTASection.astro b/site/src/components/home/CTASection.astro index 0d607e2b..6666dc7b 100644 --- a/site/src/components/home/CTASection.astro +++ b/site/src/components/home/CTASection.astro @@ -52,14 +52,14 @@ const ctas = [ ))}

- Using SynthPanel for synthetic surveys?{" "} + Using Althing for synthetic surveys?{" "} - Get SynthPanel on GitHub + Get Althing on GitHub

@@ -70,8 +70,8 @@ const ctas = [ >

Who runs this:{" "} - SynthBench is operated by DataViking, the makers of SynthPanel. - SynthPanel-branded leaderboard entries are the operator's own product. + SynthBench is operated by DataViking, the makers of Althing. + Althing-branded leaderboard entries are the operator's own product. To keep the ranking honest despite that: all scores are recomputed from published per-question data by open-source code, the underlying data is public, third-party submissions are validated by exactly the diff --git a/site/src/components/home/HeroChart.astro b/site/src/components/home/HeroChart.astro index eed6abd9..972ac7e7 100644 --- a/site/src/components/home/HeroChart.astro +++ b/site/src/components/home/HeroChart.astro @@ -164,7 +164,7 @@ const bestModelCaption = bestModels name: "Best Model", type: "bar", data: bestModels, - itemStyle: { color: "var(--color-provider-synthpanel)" }, + itemStyle: { color: "var(--color-provider-althing)" }, barMaxWidth: 50, emphasis: { focus: "series", diff --git a/site/src/components/home/SummaryLeaderboard.astro b/site/src/components/home/SummaryLeaderboard.astro index aeb9a99b..b75f63f3 100644 --- a/site/src/components/home/SummaryLeaderboard.astro +++ b/site/src/components/home/SummaryLeaderboard.astro @@ -88,8 +88,8 @@ for (const ds of leaderboard.datasets) { ensemble )} - {!entry.is_baseline && !entry.is_ensemble && entry.provider.startsWith("SynthPanel") && ( - + {!entry.is_baseline && !entry.is_ensemble && entry.provider.startsWith("Althing") && ( + conditioned )} diff --git a/site/src/components/leaderboard/DotPlot.astro b/site/src/components/leaderboard/DotPlot.astro index ad2bb6b2..76804586 100644 --- a/site/src/components/leaderboard/DotPlot.astro +++ b/site/src/components/leaderboard/DotPlot.astro @@ -95,7 +95,7 @@ const dotPlotData = models.map((m) => ({ function buildOption(models: DotModel[]) { const providerColors: Record = { - SynthPanel: "var(--color-provider-synthpanel)", + Althing: "var(--color-provider-althing)", "Raw LLM": "var(--color-provider-raw-llm)", "Random Baseline": "var(--color-provider-baseline)", }; diff --git a/site/src/components/leaderboard/LeaderboardTable.astro b/site/src/components/leaderboard/LeaderboardTable.astro index 951227c4..89b514a0 100644 --- a/site/src/components/leaderboard/LeaderboardTable.astro +++ b/site/src/components/leaderboard/LeaderboardTable.astro @@ -261,8 +261,8 @@ function hiddenInDefaultView(entry: LeaderboardEntry): boolean { )} {entry.is_ensemble && ensemble} {entry.is_baseline && baseline} - {!entry.is_baseline && entry.provider.startsWith("SynthPanel") && !entry.is_ensemble && ( - conditioned + {!entry.is_baseline && entry.provider.startsWith("Althing") && !entry.is_ensemble && ( + conditioned )} {entry.verification_badge === "verified" && ( {entry.model} {entry.is_ensemble && ensemble} {entry.is_baseline && baseline} - {!entry.is_baseline && entry.provider.startsWith("SynthPanel") && !entry.is_ensemble && ( - conditioned + {!entry.is_baseline && entry.provider.startsWith("Althing") && !entry.is_ensemble && ( + conditioned )} {entry.verification_badge === "verified" && ( ✓ verified diff --git a/site/src/components/leaderboard/VendorScorecard.astro b/site/src/components/leaderboard/VendorScorecard.astro index 39627ab5..e680fbeb 100644 --- a/site/src/components/leaderboard/VendorScorecard.astro +++ b/site/src/components/leaderboard/VendorScorecard.astro @@ -467,7 +467,7 @@ const sourceLines = Array.from(new Set(cards.map((c) => c.entry.dataset)))

No demographic conditioning data has been published for this vendor yet. The question-type matrix above shows topic-level parity; subgroup rows fill in - once SynthPanel-style conditioned runs land. + once Althing-style conditioned runs land.

)} diff --git a/site/src/components/leaderboard/WhiskerPlot.astro b/site/src/components/leaderboard/WhiskerPlot.astro index 0717987c..5186f87b 100644 --- a/site/src/components/leaderboard/WhiskerPlot.astro +++ b/site/src/components/leaderboard/WhiskerPlot.astro @@ -95,7 +95,7 @@ const whiskerData = models.map((m) => ({ function buildOption(models: WhiskerModel[]) { const providerColors: Record = { - SynthPanel: "var(--color-provider-synthpanel)", + Althing: "var(--color-provider-althing)", "Raw LLM": "var(--color-provider-raw-llm)", "Random Baseline": "var(--color-provider-baseline)", }; diff --git a/site/src/components/methodology/BaselineDefinitions.astro b/site/src/components/methodology/BaselineDefinitions.astro index 8c1acd0e..631de122 100644 --- a/site/src/components/methodology/BaselineDefinitions.astro +++ b/site/src/components/methodology/BaselineDefinitions.astro @@ -300,7 +300,7 @@ const baselines: Baseline[] = [ P_unconditioned is the SPS of the raw-LLM baseline for the same underlying model on the same dataset (the "just prompt the model" reference). Raw-LLM rows therefore resolve to 0% — - they are the reference. Product rows (e.g. conditioned SynthPanel + they are the reference. Product rows (e.g. conditioned Althing variants) show their lift above the corresponding raw model.

diff --git a/site/src/components/methodology/CostMethodology.astro b/site/src/components/methodology/CostMethodology.astro index 766b1d56..bbc484c9 100644 --- a/site/src/components/methodology/CostMethodology.astro +++ b/site/src/components/methodology/CostMethodology.astro @@ -16,19 +16,19 @@ const sections: CostSection[] = [ { id: "what-we-derive", title: "What we derive", - body: "At publish time, we multiply recorded tokens by per-model pricing from synthpanel to derive four fields per leaderboard row: cost_usd (total spend for the run), cost_per_100q (spend normalized to 100 questions), cost_per_sps_point (spend per SPS point achieved), and is_cost_estimated (true if any pricing dimension was inferred rather than measured).", + body: "At publish time, we multiply recorded tokens by per-model pricing from althing to derive four fields per leaderboard row: cost_usd (total spend for the run), cost_per_100q (spend normalized to 100 questions), cost_per_sps_point (spend per SPS point achieved), and is_cost_estimated (true if any pricing dimension was inferred rather than measured).", note: "Derivation happens in synthbench/publish.py. Tokens stay immutable in raw run JSON; cost is recomputed every time we republish, so price-list updates flow through without rerunning any benchmark.", }, { id: "pricing-snapshot", title: "Pricing snapshot", body: "The exact pricing table used to derive costs is serialized into leaderboard.json under a top-level pricing_snapshot block. It includes a snapshot_date and per-provider rates (input, output, cache-creation, and cache-read cost per million tokens). Readers can reproduce every displayed cost from the snapshot and the raw token counts.", - note: "Rates track the public price list documented on provider pricing pages (e.g. https://www.anthropic.com/pricing). The snapshot is updated when synthpanel's cost.py constants are bumped.", + note: "Rates track the public price list documented on provider pricing pages (e.g. https://www.anthropic.com/pricing). The snapshot is updated when althing's cost.py constants are bumped.", }, { id: "self-hosted-policy", title: "Self-hosted and unknown models", - body: "For self-hosted models (Ollama, local inference) and any provider whose pricing is not tracked in synthpanel, cost_usd is reported as null. We deliberately do not impute a cost — hardware, electricity, and amortization vary too widely to produce a number readers can fairly compare against API-priced rows.", + body: "For self-hosted models (Ollama, local inference) and any provider whose pricing is not tracked in althing, cost_usd is reported as null. We deliberately do not impute a cost — hardware, electricity, and amortization vary too widely to produce a number readers can fairly compare against API-priced rows.", note: "Null cost rows sort last in the leaderboard's $/100Q column and are excluded from the Cost-vs-SPS Pareto chart on the Findings page.", }, { diff --git a/site/src/components/methodology/OperatorDisclosure.astro b/site/src/components/methodology/OperatorDisclosure.astro index 5af7524a..07eb034c 100644 --- a/site/src/components/methodology/OperatorDisclosure.astro +++ b/site/src/components/methodology/OperatorDisclosure.astro @@ -1,6 +1,6 @@ --- // Operator disclosure (#316). SynthBench is run by the company that also -// ships SynthPanel, and SynthPanel-branded entries appear on the +// ships Althing, and Althing-branded entries appear on the // leaderboard. Panel testing with skeptical-executive personas showed this // is the first credibility question serious readers ask, so we answer it // affirmatively instead of leaving it to be discovered. Keep this factual: @@ -22,7 +22,7 @@ const mitigations = [ }, { title: "Private holdout applies to us too", - body: "The verified/flagged badge compares each row's public-subset score against a 20% private holdout. SynthPanel-branded rows are scored against the same hidden answer key as everyone else — and some of them are currently flagged, which we publish rather than suppress.", + body: "The verified/flagged badge compares each row's public-subset score against a 20% private holdout. Althing-branded rows are scored against the same hidden answer key as everyone else — and some of them are currently flagged, which we publish rather than suppress.", }, { title: "Unflattering results stay published", @@ -41,11 +41,11 @@ const mitigations = [ SynthBench is built and operated by DataViking, the company that also makes SynthPanel, a synthetic-survey product. SynthPanel-branded entries + >Althing, a synthetic-survey product. Althing-branded entries on the leaderboard are the operator's own product being scored on the operator's own benchmark. That is a real conflict of interest, and you should not have to take our word that the ranking is fair. diff --git a/site/src/components/methodology/RelatedWork.astro b/site/src/components/methodology/RelatedWork.astro index ae7cb901..4345edbc 100644 --- a/site/src/components/methodology/RelatedWork.astro +++ b/site/src/components/methodology/RelatedWork.astro @@ -17,7 +17,7 @@ interface Group { const groups: Group[] = [ { - heading: "SynthPanel evaluation & social simulation", + heading: "Althing evaluation & social simulation", references: [ { title: diff --git a/site/src/components/shared/Analytics.astro b/site/src/components/shared/Analytics.astro index 2288df1e..d5e5af2b 100644 --- a/site/src/components/shared/Analytics.astro +++ b/site/src/components/shared/Analytics.astro @@ -11,5 +11,5 @@ // Cross-site source attribution: stamp outbound links to sibling DataViking // properties with canonical UTMs so each destination project (separate // PostHog projects, separate cookie domains) can attribute the referral. - (function(){var SIBLINGS=["dataviking.tech","synthpanel.dev","synthbench.org","traitprint.com"],SOURCE="synthbench";function bare(x){return x.replace(/^www\./,"");}function decorate(){var self=bare(location.hostname),links=document.getElementsByTagName("a");for(var i=0;i diff --git a/site/src/data/leaderboard.json b/site/src/data/leaderboard.json index 2b2c7e6d..1caf949a 100644 --- a/site/src/data/leaderboard.json +++ b/site/src/data/leaderboard.json @@ -1,13 +1,18 @@ { - "generated_at": "2026-07-17T05:31:00.530564+00:00", + "generated_at": "2026-07-20T02:59:36.551038+00:00", "synthbench_version": "0.1.0", - "datasets": ["globalopinionqa", "gss", "opinionsqa", "subpop"], + "datasets": [ + "globalopinionqa", + "gss", + "opinionsqa", + "subpop" + ], "entries": [ { "rank": 1, "config_id": "ensemble--3-model-blend--tdefault--tplcurrent--3ba7cfb3", - "provider": "SynthPanel Ensemble (3-model)", - "model": "SynthPanel Ensemble (3-model)", + "provider": "Althing Ensemble (3-model)", + "model": "Althing Ensemble (3-model)", "provider_id": "ensemble", "model_id": "3-model-blend", "dataset": "globalopinionqa", @@ -84,12 +89,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.537, - "cost_per_100q": 0.537, - "cost_per_sps_point": 0.660143, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.761016, @@ -102,8 +107,8 @@ { "rank": 2, "config_id": "synthpanel--gpt-4o-mini--tdefault--tplcurrent--ba37570c", - "provider": "SynthPanel (GPT-4o-mini)", - "model": "SynthPanel (GPT-4o-mini)", + "provider": "Althing (GPT-4o-mini)", + "model": "Althing (GPT-4o-mini)", "provider_id": "synthpanel", "model_id": "openai/gpt-4o-mini", "dataset": "globalopinionqa", @@ -181,12 +186,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.063, - "cost_per_100q": 0.063, - "cost_per_sps_point": 0.080129, - "cost_per_response": 2.1e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.696961, @@ -371,7 +376,7 @@ "cost_usd": 0.0708, "cost_per_100q": 0.0708, "cost_per_sps_point": 0.091897, - "cost_per_response": 4.7e-5, + "cost_per_response": 4.7e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -469,7 +474,7 @@ "cost_usd": 0.02052, "cost_per_100q": 0.02052, "cost_per_sps_point": 0.026918, - "cost_per_response": 1.4e-5, + "cost_per_response": 1.4e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -485,8 +490,8 @@ { "rank": 6, "config_id": "synthpanel--gemini-2.5-flash-lite--tdefault--tplcurrent--ca4dfebb", - "provider": "SynthPanel (Gemini Flash Lite)", - "model": "SynthPanel (Gemini Flash Lite)", + "provider": "Althing (Gemini Flash Lite)", + "model": "Althing (Gemini Flash Lite)", "provider_id": "synthpanel", "model_id": "google/gemini-2.5-flash-lite", "dataset": "globalopinionqa", @@ -564,12 +569,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.042, - "cost_per_100q": 0.042, - "cost_per_sps_point": 0.055147, - "cost_per_response": 1.4e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.677469, @@ -657,7 +662,7 @@ "cost_usd": 0.017308, "cost_per_100q": 0.017308, "cost_per_sps_point": 0.022737, - "cost_per_response": 1.2e-5, + "cost_per_response": 1.2e-05, "tokens_per_response": 112.13, "is_cost_estimated": false, "latency_p50_seconds": 0.6102, @@ -755,7 +760,7 @@ "cost_usd": 0.0315, "cost_per_100q": 0.0315, "cost_per_sps_point": 0.04204, - "cost_per_response": 2.1e-5, + "cost_per_response": 2.1e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -771,8 +776,8 @@ { "rank": 9, "config_id": "synthpanel--llama-3.3-70b-instruct--tdefault--tplcurrent--4520a502", - "provider": "SynthPanel (Llama 3.3 70B)", - "model": "SynthPanel (Llama 3.3 70B)", + "provider": "Althing (Llama 3.3 70B)", + "model": "Althing (Llama 3.3 70B)", "provider_id": "synthpanel", "model_id": "meta-llama/llama-3.3-70b-instruct", "dataset": "globalopinionqa", @@ -842,12 +847,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.039254, - "cost_per_100q": 0.039254, - "cost_per_sps_point": 0.052676, - "cost_per_response": 2.6e-5, - "tokens_per_response": 178.62, - "is_cost_estimated": false, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, + "tokens_per_response": null, + "is_cost_estimated": null, "latency_p50_seconds": 13.7587, "latency_p95_seconds": 26.5763, "sps_public": 0.659478, @@ -1049,8 +1054,8 @@ { "rank": 12, "config_id": "synthpanel--claude-haiku-4-5--tdefault--tplcurrent--49fbdb2d", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "globalopinionqa", @@ -1128,12 +1133,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.432, - "cost_per_100q": 0.432, - "cost_per_sps_point": 0.595981, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.651432, @@ -1244,8 +1249,8 @@ { "rank": 1, "config_id": "synthpanel--claude-haiku-4-5--tdefault--tplcurrent--776efb01", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "gss", @@ -1323,26 +1328,26 @@ } }, "demographic_scorecard": null, - "cost_usd": 1.667205, - "cost_per_100q": 2.22294, - "cost_per_sps_point": 2.075722, - "cost_per_response": 0.000741, - "tokens_per_response": 258.2, - "is_cost_estimated": false, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, + "tokens_per_response": null, + "is_cost_estimated": null, "latency_p50_seconds": 18.1848, "latency_p95_seconds": 68.5154, "sps_public": 0.727978, "sps_private": 0.681912, "sps_public_private_delta": 0.046066, "verification_badge": "verified", - "run_count": 1, + "run_count": 2, "dataset_coverage_count": 4 }, { "rank": 2, "config_id": "synthpanel--claude-haiku-4-5--tdefault--tplstructured--fd829138", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "gss", @@ -1421,19 +1426,19 @@ } }, "demographic_scorecard": null, - "cost_usd": 2.239675, - "cost_per_100q": 2.986233, - "cost_per_sps_point": 2.829777, - "cost_per_response": 0.000995, - "tokens_per_response": 857.25, - "is_cost_estimated": false, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, + "tokens_per_response": null, + "is_cost_estimated": null, "latency_p50_seconds": 6.6938, "latency_p95_seconds": 26.249, "sps_public": 0.717715, "sps_private": 0.635018, "sps_public_private_delta": 0.082696, "verification_badge": "flagged", - "run_count": 1, + "run_count": 2, "dataset_coverage_count": 1 }, { @@ -1521,7 +1526,7 @@ "cost_usd": 0.080309, "cost_per_100q": 0.107078, "cost_per_sps_point": 0.101739, - "cost_per_response": 3.6e-5, + "cost_per_response": 3.6e-05, "tokens_per_response": 111.45, "is_cost_estimated": false, "latency_p50_seconds": 1.2907, @@ -1633,8 +1638,8 @@ { "rank": 1, "config_id": "ensemble--3-model-blend--tdefault--tplcurrent--1bef3e62", - "provider": "SynthPanel Ensemble (3-model)", - "model": "SynthPanel Ensemble (3-model)", + "provider": "Althing Ensemble (3-model)", + "model": "Althing Ensemble (3-model)", "provider_id": "ensemble", "model_id": "3-model-blend", "dataset": "opinionsqa", @@ -1735,12 +1740,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 6.1218, - "cost_per_100q": 0.895, - "cost_per_sps_point": 6.982609, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.833043, @@ -1859,7 +1864,7 @@ "cost_usd": 1.61424, "cost_per_100q": 0.236, "cost_per_sps_point": 1.946716, - "cost_per_response": 4.7e-5, + "cost_per_response": 4.7e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -1875,8 +1880,8 @@ { "rank": 3, "config_id": "synthpanel--claude-sonnet-4--tdefault--tplcurrent--8d1cda33", - "provider": "SynthPanel (Sonnet 4)", - "model": "SynthPanel (Sonnet 4)", + "provider": "Althing (Sonnet 4)", + "model": "Althing (Sonnet 4)", "provider_id": "synthpanel", "model_id": "anthropic/claude-sonnet-4", "dataset": "opinionsqa", @@ -1978,12 +1983,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 8.86464, - "cost_per_100q": 1.296, - "cost_per_sps_point": 10.69325, - "cost_per_response": 0.000432, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.757878, @@ -1997,8 +2002,8 @@ { "rank": 4, "config_id": "synthpanel--claude-haiku-4-5--tdefault--tplcurrent--555a9f1e", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "opinionsqa", @@ -2100,12 +2105,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 4.9248, - "cost_per_100q": 0.72, - "cost_per_sps_point": 5.941046, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.763167, @@ -2119,8 +2124,8 @@ { "rank": 5, "config_id": "synthpanel--gpt-4o-mini--tdefault--tplcurrent--b1d864c3", - "provider": "SynthPanel (GPT-4o-mini)", - "model": "SynthPanel (GPT-4o-mini)", + "provider": "Althing (GPT-4o-mini)", + "model": "Althing (GPT-4o-mini)", "provider_id": "synthpanel", "model_id": "openai/gpt-4o-mini", "dataset": "opinionsqa", @@ -2222,12 +2227,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.7182, - "cost_per_100q": 0.105, - "cost_per_sps_point": 0.873153, - "cost_per_response": 2.1e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.737558, @@ -2347,7 +2352,7 @@ "cost_usd": 0.467856, "cost_per_100q": 0.0684, "cost_per_sps_point": 0.571267, - "cost_per_response": 1.4e-5, + "cost_per_response": 1.4e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -2363,8 +2368,8 @@ { "rank": 7, "config_id": "synthpanel--gemini-2.5-flash-lite--tdefault--tplcurrent--5a08bef0", - "provider": "SynthPanel (Gemini Flash Lite)", - "model": "SynthPanel (Gemini Flash Lite)", + "provider": "Althing (Gemini Flash Lite)", + "model": "Althing (Gemini Flash Lite)", "provider_id": "synthpanel", "model_id": "google/gemini-2.5-flash-lite", "dataset": "opinionsqa", @@ -2466,12 +2471,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.4788, - "cost_per_100q": 0.07, - "cost_per_sps_point": 0.586736, - "cost_per_response": 1.4e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.755437, @@ -2835,7 +2840,7 @@ "cost_usd": 0.7182, "cost_per_100q": 0.105, "cost_per_sps_point": 0.883919, - "cost_per_response": 2.1e-5, + "cost_per_response": 2.1e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -2851,8 +2856,8 @@ { "rank": 11, "config_id": "synthpanel--llama-3.3-70b-instruct--tdefault--tplcurrent--482313b7", - "provider": "SynthPanel (Llama 3.3 70B)", - "model": "SynthPanel (Llama 3.3 70B)", + "provider": "Althing (Llama 3.3 70B)", + "model": "Althing (Llama 3.3 70B)", "provider_id": "synthpanel", "model_id": "meta-llama/llama-3.3-70b-instruct", "dataset": "opinionsqa", @@ -2946,12 +2951,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.0358, - "cost_per_100q": 0.0358, - "cost_per_sps_point": 0.045263, - "cost_per_response": 2.4e-5, - "tokens_per_response": 164.52, - "is_cost_estimated": false, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, + "tokens_per_response": null, + "is_cost_estimated": null, "latency_p50_seconds": 11.3475, "latency_p95_seconds": 17.887, "sps_public": 0.712898, @@ -3290,7 +3295,7 @@ "cost_usd": 0.042, "cost_per_100q": 0.042, "cost_per_sps_point": 0.056923, - "cost_per_response": 1.4e-5, + "cost_per_response": 1.4e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -3427,8 +3432,8 @@ { "rank": 1, "config_id": "ensemble--3-model-blend--tdefault--tplcurrent--ef75a647", - "provider": "SynthPanel Ensemble (3-model)", - "model": "SynthPanel Ensemble (3-model)", + "provider": "Althing Ensemble (3-model)", + "model": "Althing Ensemble (3-model)", "provider_id": "ensemble", "model_id": "3-model-blend", "dataset": "subpop", @@ -3529,12 +3534,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 1.074, - "cost_per_100q": 0.537, - "cost_per_sps_point": 1.25157, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.796137, @@ -3547,8 +3552,8 @@ { "rank": 2, "config_id": "synthpanel--gemini-2.5-flash-lite--tdefault--tplcurrent--650ec804", - "provider": "SynthPanel (Gemini Flash Lite)", - "model": "SynthPanel (Gemini Flash Lite)", + "provider": "Althing (Gemini Flash Lite)", + "model": "Althing (Gemini Flash Lite)", "provider_id": "synthpanel", "model_id": "google/gemini-2.5-flash-lite", "dataset": "subpop", @@ -3828,12 +3833,12 @@ } ] }, - "cost_usd": 0.084, - "cost_per_100q": 0.042, - "cost_per_sps_point": 0.102315, - "cost_per_response": 1.4e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.742098, @@ -3847,8 +3852,8 @@ { "rank": 3, "config_id": "synthpanel--claude-haiku-4-5--tdefault--tplcurrent--fe74b735", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "subpop", @@ -4275,12 +4280,12 @@ } ] }, - "cost_usd": 0.864, - "cost_per_100q": 0.432, - "cost_per_sps_point": 1.067877, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.724199, @@ -4578,7 +4583,7 @@ "cost_usd": 0.08208, "cost_per_100q": 0.04104, "cost_per_sps_point": 0.103179, - "cost_per_response": 1.4e-5, + "cost_per_response": 1.4e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -4594,8 +4599,8 @@ { "rank": 5, "config_id": "synthpanel--gpt-4o-mini--tdefault--tplcurrent--b631856b", - "provider": "SynthPanel (GPT-4o-mini)", - "model": "SynthPanel (GPT-4o-mini)", + "provider": "Althing (GPT-4o-mini)", + "model": "Althing (GPT-4o-mini)", "provider_id": "synthpanel", "model_id": "openai/gpt-4o-mini", "dataset": "subpop", @@ -4875,12 +4880,12 @@ } ] }, - "cost_usd": 0.126, - "cost_per_100q": 0.063, - "cost_per_sps_point": 0.160147, - "cost_per_response": 2.1e-5, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.692576, @@ -5170,7 +5175,7 @@ "cost_usd": 0.0708, "cost_per_100q": 0.0708, "cost_per_sps_point": 0.090471, - "cost_per_response": 4.7e-5, + "cost_per_response": 4.7e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -5284,7 +5289,7 @@ "cost_usd": 0.016446, "cost_per_100q": 0.016446, "cost_per_sps_point": 0.021185, - "cost_per_response": 1.1e-5, + "cost_per_response": 1.1e-05, "tokens_per_response": 106.62, "is_cost_estimated": false, "latency_p50_seconds": 0.5617, @@ -5300,8 +5305,8 @@ { "rank": 8, "config_id": "synthpanel--llama-3.3-70b-instruct--tdefault--tplcurrent--9d6e5c2c", - "provider": "SynthPanel (Llama 3.3 70B)", - "model": "SynthPanel (Llama 3.3 70B)", + "provider": "Althing (Llama 3.3 70B)", + "model": "Althing (Llama 3.3 70B)", "provider_id": "synthpanel", "model_id": "meta-llama/llama-3.3-70b-instruct", "dataset": "subpop", @@ -5395,12 +5400,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.0355, - "cost_per_100q": 0.0355, - "cost_per_sps_point": 0.04574, - "cost_per_response": 2.4e-5, - "tokens_per_response": 166.04, - "is_cost_estimated": false, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, + "tokens_per_response": null, + "is_cost_estimated": null, "latency_p50_seconds": 9.3675, "latency_p95_seconds": 18.7843, "sps_public": 0.689315, @@ -5414,8 +5419,8 @@ { "rank": 9, "config_id": "synthpanel--claude-haiku-4-5--t0.85--tplcurrent--b8092537", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "subpop", @@ -5511,12 +5516,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.432, - "cost_per_100q": 0.432, - "cost_per_sps_point": 0.560257, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": 8.0217, "latency_p95_seconds": 9.6137, "sps_public": 0.673672, @@ -5806,7 +5811,7 @@ "cost_usd": 0.0315, "cost_per_100q": 0.0315, "cost_per_sps_point": 0.040902, - "cost_per_response": 2.1e-5, + "cost_per_response": 2.1e-05, "tokens_per_response": null, "is_cost_estimated": true, "latency_p50_seconds": null, @@ -6162,8 +6167,8 @@ { "rank": 14, "config_id": "synthpanel--claude-haiku-4-5--t0.85--tplminimal--7cffeaef", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "subpop", @@ -6251,12 +6256,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.432, - "cost_per_100q": 0.432, - "cost_per_sps_point": 0.739111, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.600608, @@ -6270,8 +6275,8 @@ { "rank": 15, "config_id": "synthpanel--claude-haiku-4-5--t0.85--tplvalues--55360dd0", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "subpop", @@ -6359,12 +6364,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.432, - "cost_per_100q": 0.432, - "cost_per_sps_point": 0.747899, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.621626, @@ -6378,8 +6383,8 @@ { "rank": 16, "config_id": "synthpanel--claude-haiku-4-5--t0.85--tpldemo--553c822b", - "provider": "SynthPanel (Haiku 4.5)", - "model": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "provider_id": "synthpanel", "model_id": "anthropic/claude-haiku-4-5", "dataset": "subpop", @@ -6467,12 +6472,12 @@ } }, "demographic_scorecard": null, - "cost_usd": 0.432, - "cost_per_100q": 0.432, - "cost_per_sps_point": 0.757873, - "cost_per_response": 0.000144, + "cost_usd": null, + "cost_per_100q": null, + "cost_per_sps_point": null, + "cost_per_response": null, "tokens_per_response": null, - "is_cost_estimated": true, + "is_cost_estimated": null, "latency_p50_seconds": null, "latency_p95_seconds": null, "sps_public": 0.615818, @@ -7001,6 +7006,12 @@ "rep_count": 30, "sps": 0.7188 }, + { + "model": "synthpanel/openrouter/anthropic/claude-haiku-4-5", + "dataset": "opinionsqa", + "rep_count": 30, + "sps": 0.7233 + }, { "model": "synthpanel/openrouter/anthropic/claude-haiku-4-5", "dataset": "opinionsqa", @@ -7576,7 +7587,7 @@ "ensemble_comparison": [ { "dataset": "globalopinionqa", - "best_single_model": "SynthPanel (GPT-4o-mini)", + "best_single_model": "Althing (GPT-4o-mini)", "best_single_framework": "product", "best_single_sps": 0.786229, "ensemble_sps": 0.81346, @@ -7598,7 +7609,7 @@ }, { "dataset": "subpop", - "best_single_model": "SynthPanel (Gemini Flash Lite)", + "best_single_model": "Althing (Gemini Flash Lite)", "best_single_framework": "product", "best_single_sps": 0.820993, "ensemble_sps": 0.858122, @@ -7613,26 +7624,26 @@ "dataset": "globalopinionqa", "n_questions": 100, "constituents": [ - "SynthPanel (Haiku 4.5)", - "SynthPanel (Gemini Flash Lite)", - "SynthPanel (GPT-4o-mini)" + "Althing (Haiku 4.5)", + "Althing (Gemini Flash Lite)", + "Althing (GPT-4o-mini)" ], "pairs": [ { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (Gemini Flash Lite)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (Gemini Flash Lite)", "pearson_r": 0.41888, "n": 100 }, { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.307716, "n": 100 }, { - "a": "SynthPanel (Gemini Flash Lite)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Gemini Flash Lite)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.49332, "n": 100 } @@ -7643,26 +7654,26 @@ "dataset": "opinionsqa", "n_questions": 684, "constituents": [ - "SynthPanel (Haiku 4.5)", - "SynthPanel (Gemini Flash Lite)", - "SynthPanel (GPT-4o-mini)" + "Althing (Haiku 4.5)", + "Althing (Gemini Flash Lite)", + "Althing (GPT-4o-mini)" ], "pairs": [ { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (Gemini Flash Lite)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (Gemini Flash Lite)", "pearson_r": 0.223567, "n": 684 }, { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.355047, "n": 684 }, { - "a": "SynthPanel (Gemini Flash Lite)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Gemini Flash Lite)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.307715, "n": 684 } @@ -7673,26 +7684,26 @@ "dataset": "subpop", "n_questions": 200, "constituents": [ - "SynthPanel (Haiku 4.5)", - "SynthPanel (Gemini Flash Lite)", - "SynthPanel (GPT-4o-mini)" + "Althing (Haiku 4.5)", + "Althing (Gemini Flash Lite)", + "Althing (GPT-4o-mini)" ], "pairs": [ { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (Gemini Flash Lite)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (Gemini Flash Lite)", "pearson_r": 0.282243, "n": 200 }, { - "a": "SynthPanel (Haiku 4.5)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Haiku 4.5)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.259842, "n": 200 }, { - "a": "SynthPanel (Gemini Flash Lite)", - "b": "SynthPanel (GPT-4o-mini)", + "a": "Althing (Gemini Flash Lite)", + "b": "Althing (GPT-4o-mini)", "pearson_r": 0.424602, "n": 200 } @@ -7897,7 +7908,7 @@ ], "elicitation_comparison": [ { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "framework": "product", "dataset": "gss", "elicitation": "structured", @@ -7972,7 +7983,7 @@ ] }, { - "provider": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", "framework": "product", "template": "structured", "dataset": "gss", @@ -8014,7 +8025,7 @@ ] }, { - "provider": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", "framework": "product", "template": null, "dataset": "gss", @@ -8131,7 +8142,7 @@ "comparison_sets": { "ensemble_comparison": "best_single = highest recomputed-SPS non-ensemble, non-baseline leaderboard row (raw or product framework) evaluated on at least as many questions as the ensemble, after per-(model, framework, dataset) dedup; random_baseline = recomputed SPS of the random-baseline run for the same dataset (n in random_baseline_n).", "ensemble_error_correlation": "For each dataset's published ensemble: pairwise Pearson r between the constituent runs' committed per-question JSD vectors (each run's error magnitude vs the canonical human distribution), over the ensemble's common question set, read from the exact files named in ensemble_sources. Measures whether constituents err on the same questions. Signed per-option residual correlations need the stripped human_distribution fields (#308) and are carried as the ensemble_signed_error_correlation asserted constant.", - "temperature_sweep": "OpinionsQA sweep-tagged SynthPanel runs (' t=' provider suffix); mean/std of recomputed SPS across the n_runs replications per (model, temperature) cell.", + "temperature_sweep": "OpinionsQA sweep-tagged Althing runs (' t=' provider suffix); mean/std of recomputed SPS across the n_runs replications per (model, temperature) cell.", "conditioning_results": "Per-group conditioned SubPOP evaluations (Haiku 4.5, t=0.85); mean p_dist / p_cond across n_replications runs carrying a demographic_breakdown for the group.", "template_comparison": "SubPOP persona-template variant runs (Haiku 4.5, t=0.85, 100 questions); mean/std of recomputed SPS across n_runs.", "elicitation_comparison": "Matched pairs of deduped runs differing only in elicitation mode: a run whose config carries an explicit elicitation template variant (e.g. tpl=structured, schema-forced capture) vs the natural-elicitation run of the identical configuration (same base provider, dataset, samples_per_question, question_set_hash, temperature, effort, and question count). All scores recomputed from per-question rows; deltas are structured minus natural.", @@ -26067,12 +26078,54 @@ "Llama 3.3 70B" ], "matrix": [ - [0.0, 0.335683, 0.27869, 0.269583, 0.294958, 0.311335], - [0.335683, 0.0, 0.324039, 0.326184, 0.314417, 0.275544], - [0.27869, 0.324039, 0.0, 0.277088, 0.261902, 0.21221], - [0.269583, 0.326184, 0.277088, 0.0, 0.187054, 0.282885], - [0.294958, 0.314417, 0.261902, 0.187054, 0.0, 0.293885], - [0.311335, 0.275544, 0.21221, 0.282885, 0.293885, 0.0] + [ + 0.0, + 0.335683, + 0.27869, + 0.269583, + 0.294958, + 0.311335 + ], + [ + 0.335683, + 0.0, + 0.324039, + 0.326184, + 0.314417, + 0.275544 + ], + [ + 0.27869, + 0.324039, + 0.0, + 0.277088, + 0.261902, + 0.21221 + ], + [ + 0.269583, + 0.326184, + 0.277088, + 0.0, + 0.187054, + 0.282885 + ], + [ + 0.294958, + 0.314417, + 0.261902, + 0.187054, + 0.0, + 0.293885 + ], + [ + 0.311335, + 0.275544, + 0.21221, + 0.282885, + 0.293885, + 0.0 + ] ], "mean_cross_model_jsd": 0.28303, "mean_human_jsd": 0.3612 @@ -26088,13 +26141,69 @@ "Llama 3.3 70B" ], "matrix": [ - [0.0, 0.254674, 0.29902, 0.303531, 0.297094, 0.309816, 0.273953], - [0.254674, 0.0, 0.336434, 0.402281, 0.354759, 0.403376, 0.379415], - [0.29902, 0.336434, 0.0, 0.161905, 0.219506, 0.228906, 0.229255], - [0.303531, 0.402281, 0.161905, 0.0, 0.219523, 0.195565, 0.204973], - [0.297094, 0.354759, 0.219506, 0.219523, 0.0, 0.218144, 0.254065], - [0.309816, 0.403376, 0.228906, 0.195565, 0.218144, 0.0, 0.342674], - [0.273953, 0.379415, 0.229255, 0.204973, 0.254065, 0.342674, 0.0] + [ + 0.0, + 0.254674, + 0.29902, + 0.303531, + 0.297094, + 0.309816, + 0.273953 + ], + [ + 0.254674, + 0.0, + 0.336434, + 0.402281, + 0.354759, + 0.403376, + 0.379415 + ], + [ + 0.29902, + 0.336434, + 0.0, + 0.161905, + 0.219506, + 0.228906, + 0.229255 + ], + [ + 0.303531, + 0.402281, + 0.161905, + 0.0, + 0.219523, + 0.195565, + 0.204973 + ], + [ + 0.297094, + 0.354759, + 0.219506, + 0.219523, + 0.0, + 0.218144, + 0.254065 + ], + [ + 0.309816, + 0.403376, + 0.228906, + 0.195565, + 0.218144, + 0.0, + 0.342674 + ], + [ + 0.273953, + 0.379415, + 0.229255, + 0.204973, + 0.254065, + 0.342674, + 0.0 + ] ], "mean_cross_model_jsd": 0.280422, "mean_human_jsd": 0.297788 @@ -26108,19 +26217,49 @@ "Llama 3.3 70B" ], "matrix": [ - [0.0, 0.381428, 0.24479, 0.26794, 0.303028], - [0.381428, 0.0, 0.330614, 0.291416, 0.31692], - [0.24479, 0.330614, 0.0, 0.186912, 0.30062], - [0.26794, 0.291416, 0.186912, 0.0, 0.360195], - [0.303028, 0.31692, 0.30062, 0.360195, 0.0] + [ + 0.0, + 0.381428, + 0.24479, + 0.26794, + 0.303028 + ], + [ + 0.381428, + 0.0, + 0.330614, + 0.291416, + 0.31692 + ], + [ + 0.24479, + 0.330614, + 0.0, + 0.186912, + 0.30062 + ], + [ + 0.26794, + 0.291416, + 0.186912, + 0.0, + 0.360195 + ], + [ + 0.303028, + 0.31692, + 0.30062, + 0.360195, + 0.0 + ] ], "mean_cross_model_jsd": 0.298386, "mean_human_jsd": 0.355599 } }, "pricing_snapshot": { - "generated_at": "2026-07-17T05:31:00.543773+00:00", - "synth_panel_version": "1.5.7", + "generated_at": "2026-07-20T02:59:36.572458+00:00", + "althing_version": "1.6.0", "snapshot_date": "2026-05-10", "rates": { "haiku": { @@ -26136,10 +26275,10 @@ "cache_read_cost_per_million": 0.3 }, "opus": { - "input_cost_per_million": 15.0, - "output_cost_per_million": 75.0, - "cache_creation_cost_per_million": 18.75, - "cache_read_cost_per_million": 1.5 + "input_cost_per_million": 5.0, + "output_cost_per_million": 25.0, + "cache_creation_cost_per_million": 6.25, + "cache_read_cost_per_million": 0.5 }, "gemini-2.5-pro": { "input_cost_per_million": 1.25, diff --git a/site/src/lib/vendor-slug.ts b/site/src/lib/vendor-slug.ts index 55955c6b..6aaa4f69 100644 --- a/site/src/lib/vendor-slug.ts +++ b/site/src/lib/vendor-slug.ts @@ -1,6 +1,6 @@ // Stable URL slug for vendor (provider) names on /leaderboard// pages. // Provider strings in leaderboard.json contain spaces, dots, and parentheses -// (e.g. "SynthPanel (Haiku 4.5)") — none safe in a path segment. We normalize +// (e.g. "Althing (Haiku 4.5)") — none safe in a path segment. We normalize // by lowercasing, stripping anything outside [a-z0-9], and collapsing runs of // dashes so the slug round-trips cleanly through static-path generation. diff --git a/site/src/pages/compare.astro b/site/src/pages/compare.astro index 0297d38b..b49f1742 100644 --- a/site/src/pages/compare.astro +++ b/site/src/pages/compare.astro @@ -274,7 +274,7 @@ const exploreUrl = `${base}/explore`; "var(--color-accent)", "var(--color-provider-raw-llm)", "var(--color-baseline)", - "var(--color-provider-synthpanel)", + "var(--color-provider-althing)", "var(--color-muted)", ]; diff --git a/site/src/pages/config/[id].astro b/site/src/pages/config/[id].astro index 08830acb..c48a23fa 100644 --- a/site/src/pages/config/[id].astro +++ b/site/src/pages/config/[id].astro @@ -358,7 +358,7 @@ const base = rawBase.endsWith("/") ? rawBase : `${rawBase}/`; const styles = getComputedStyle(document.documentElement); const barColor = - styles.getPropertyValue("--color-provider-synthpanel").trim() || "#4f46e5"; + styles.getPropertyValue("--color-provider-althing").trim() || "#4f46e5"; const barData = means.map((m) => (m === null ? { value: 0, itemStyle: { opacity: 0.15 } } : m)); diff --git a/site/src/pages/data/leaderboard.json.ts b/site/src/pages/data/leaderboard.json.ts index 9509e43f..81e83936 100644 --- a/site/src/pages/data/leaderboard.json.ts +++ b/site/src/pages/data/leaderboard.json.ts @@ -2,7 +2,7 @@ import leaderboardRaw from "@/data/leaderboard.json"; import type { LeaderboardData } from "@/types/leaderboard"; import type { APIRoute } from "astro"; -// sb-ty6 / closes #295 — public stable JSON contract for SynthPanel's +// sb-ty6 / closes #295 — public stable JSON contract for Althing's // `--best-model-for` recommendation lookup and any downstream consumer that // wants the live leaderboard without scraping the site or cloning the repo. // @@ -21,7 +21,7 @@ import type { APIRoute } from "astro"; // (Cloudflare Pages). Current policy: 1h browser cache, 6h edge cache, // stale-while-revalidate up to 24h. Clients SHOULD honor Cache-Control. // -// SynthPanel's fallback behavior (sy-nkh) treats any non-200 as "use the +// Althing's fallback behavior (sy-nkh) treats any non-200 as "use the // bundled snapshot," so a stale endpoint is still safe — just stale. Removing // or renaming this URL would break that fallback, so it MUST stay reachable // at this exact path even if the underlying payload schema evolves. diff --git a/site/src/pages/question/[dataset]/[key].astro b/site/src/pages/question/[dataset]/[key].astro index 90050740..a7ce9c0e 100644 --- a/site/src/pages/question/[dataset]/[key].astro +++ b/site/src/pages/question/[dataset]/[key].astro @@ -142,7 +142,7 @@ const BAR_COLORS = [ "var(--color-accent)", "var(--color-provider-raw-llm)", "var(--color-baseline)", - "var(--color-provider-synthpanel)", + "var(--color-provider-althing)", "var(--color-muted)", ]; @@ -631,7 +631,7 @@ const encodePathSegment = (s: string): string => "var(--color-accent)", "var(--color-provider-raw-llm)", "var(--color-baseline)", - "var(--color-provider-synthpanel)", + "var(--color-provider-althing)", "var(--color-muted)", ]; diff --git a/site/src/pages/run/[id].astro b/site/src/pages/run/[id].astro index 51b797fb..11f058f7 100644 --- a/site/src/pages/run/[id].astro +++ b/site/src/pages/run/[id].astro @@ -247,7 +247,7 @@ const questionBaseUrl = `${base}/question/${encodePathSegment(entry.dataset)}`; "var(--color-accent)", "var(--color-provider-raw-llm)", "var(--color-baseline)", - "var(--color-provider-synthpanel)", + "var(--color-provider-althing)", "var(--color-muted)", ]; diff --git a/site/src/styles/global.css b/site/src/styles/global.css index d2a05a56..c60b3f08 100644 --- a/site/src/styles/global.css +++ b/site/src/styles/global.css @@ -15,7 +15,7 @@ --color-muted-dark: oklch(0.72 0.01 264); --color-baseline: oklch(0.55 0.14 30); --color-best: oklch(0.55 0.18 264); - --color-provider-synthpanel: #4f46e5; + --color-provider-althing: #4f46e5; --color-provider-raw-llm: #0891b2; --color-provider-baseline: #a3a3a3; /* Page background — paired with `color-scheme` so iOS Safari overscroll @@ -49,7 +49,7 @@ body { --color-muted: var(--color-muted-dark); --color-accent: oklch(0.78 0.18 150); --color-baseline: oklch(0.78 0.12 30); - --color-provider-synthpanel: #a5b4fc; + --color-provider-althing: #a5b4fc; --color-provider-raw-llm: #67e8f9; } /* In dark mode, `text-primary` (used for links and accent text) needs a diff --git a/site/src/types/leaderboard.ts b/site/src/types/leaderboard.ts index 186c22d1..be04536e 100644 --- a/site/src/types/leaderboard.ts +++ b/site/src/types/leaderboard.ts @@ -20,10 +20,10 @@ export interface LeaderboardEntry { provider: string; model: string; /** - * Machine-runnable execution-layer ID: "raw" | "synthpanel" | "ensemble" | + * Machine-runnable execution-layer ID: "raw" | "althing" | "ensemble" | * "baseline". Unlike `provider`/`model` (display labels), this is stable and * parseable — it tells a consumer whether a score reflects a raw model, the - * SynthPanel product layer, an ensemble, or a statistical baseline. Emitted + * Althing product layer, an ensemble, or a statistical baseline. Emitted * by publish.py via synthbench.config_id.runnable_ids (sb-7ly). */ provider_id?: string; @@ -470,12 +470,12 @@ export interface Baselines { /** * Runtime pricing manifest captured by publish.py at publish time (sb-tbm - * Slice 3). Documents which synthpanel pricing rates were applied to cost + * Slice 3). Documents which althing pricing rates were applied to cost * fields in this leaderboard build. */ export interface PricingSnapshot { generated_at: string; - synth_panel_version: string; + althing_version: string; snapshot_date: string; rates: Record>; } diff --git a/site/src/types/runs.ts b/site/src/types/runs.ts index db97dbbc..e6a35f1e 100644 --- a/site/src/types/runs.ts +++ b/site/src/types/runs.ts @@ -13,7 +13,7 @@ import type { DemographicBreakdown, TopicCategory } from "./leaderboard"; -export type RunFramework = "synthpanel" | "openrouter" | "raw" | "ensemble" | "baseline" | string; +export type RunFramework = "althing" | "openrouter" | "raw" | "ensemble" | "baseline" | string; export type Dataset = "opinionsqa" | "globalopinionqa" | "subpop" | string; diff --git a/src/synthbench/cli.py b/src/synthbench/cli.py index 31a0a5a4..0d42d0b0 100644 --- a/src/synthbench/cli.py +++ b/src/synthbench/cli.py @@ -37,7 +37,15 @@ # Providers whose constructor accepts a model identifier / temperature. _MODEL_PROVIDERS = frozenset( - {"raw-anthropic", "raw-openai", "raw-gemini", "openrouter", "ollama", "synthpanel"} + { + "raw-anthropic", + "raw-openai", + "raw-gemini", + "openrouter", + "ollama", + "althing", + "synthpanel", + } ) # Providers that can actually honour a reasoning-effort level. Everything @@ -90,23 +98,23 @@ def _provider_kwargs( ) kwargs["temperature"] = temperature if prompt_template is not None: - if provider_name != "synthpanel": + if provider_name not in ("althing", "synthpanel"): raise click.UsageError( - "--prompt-template is only supported by the synthpanel provider." + "--prompt-template is only supported by the althing provider." ) kwargs["prompt_template"] = prompt_template if elicitation is not None: - if provider_name != "synthpanel": + if provider_name not in ("althing", "synthpanel"): raise click.UsageError( - "--elicitation is only supported by the synthpanel provider." + "--elicitation is only supported by the althing provider." ) kwargs["elicitation"] = elicitation if effort is not None: - if provider_name == "synthpanel": + if provider_name in ("althing", "synthpanel"): raise click.UsageError( - "--effort is not supported by provider 'synthpanel': the " - "synthpanel CLI does not yet expose a reasoning-effort knob, " - "so the run could not actually honour it. Once synthpanel " + "--effort is not supported by provider 'althing': the " + "althing CLI does not yet expose a reasoning-effort knob, " + "so the run could not actually honour it. Once althing " "grows one, wire it through here rather than tagging runs " "with metadata that never applied." ) @@ -140,7 +148,7 @@ def main(): "--provider", "-p", required=True, - help="Provider name (raw-anthropic, raw-openai, raw-gemini, openrouter, ollama, synthpanel, http).", + help="Provider name (raw-anthropic, raw-openai, raw-gemini, openrouter, ollama, althing, http).", ) @click.option( "--model", @@ -247,14 +255,14 @@ def main(): "--prompt-template", type=click.Path(exists=True), default=None, - help="Path to a custom persona prompt template file (synthpanel only).", + help="Path to a custom persona prompt template file (althing only).", ) @click.option( "--elicitation", type=click.Choice(["natural", "structured"]), default=None, help=( - "Elicitation mode (synthpanel only). 'natural' (default) parses " + "Elicitation mode (althing only). 'natural' (default) parses " "prose responses; 'structured' forces a schema-constrained tool " "call (enum of the option strings) so no prose parsing occurs. " "Structured runs are stamped as a template variant " diff --git a/src/synthbench/config_id.py b/src/synthbench/config_id.py index 8ee4e796..f83092d7 100644 --- a/src/synthbench/config_id.py +++ b/src/synthbench/config_id.py @@ -1,7 +1,7 @@ """Canonical config ID parsing for the run explorer. Turns raw provider strings like -``synthpanel/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current`` +``althing/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current`` into a structured record (framework / base_provider / model / knobs) and a stable slug ``framework--model--t--tpl[--eff]--`` suitable for use as a URL path segment (the ``eff`` segment appears only @@ -34,7 +34,7 @@ } # Model-name prefix → canonical vendor. Used to infer base_provider when the -# path itself doesn't carry a vendor segment (e.g. ``synthpanel/claude-...``). +# path itself doesn't carry a vendor segment (e.g. ``althing/claude-...``). _MODEL_VENDOR_PREFIXES: tuple[tuple[str, str], ...] = ( ("claude-", "anthropic"), ("gemini-", "google"), @@ -92,8 +92,8 @@ def as_canonical_dict(self) -> dict[str, Any]: def _split_provider_and_knobs(provider: str) -> tuple[str, list[str]]: """Split the whitespace-separated provider/knob form. - ``synthpanel/.../claude-haiku-4-5 t=0.85 tpl=current`` → - (``synthpanel/.../claude-haiku-4-5``, [``t=0.85``, ``tpl=current``]). + ``althing/.../claude-haiku-4-5 t=0.85 tpl=current`` → + (``althing/.../claude-haiku-4-5``, [``t=0.85``, ``tpl=current``]). """ tokens = provider.strip().split() if not tokens: @@ -143,11 +143,11 @@ def _parse_path(path: str) -> tuple[str, str | None, str]: raw-anthropic/claude-haiku-4-5 → ("raw", "anthropic", "claude-haiku-4-5") raw-gemini/gemini-2.5-flash-lite → ("raw", "google", "gemini-2.5-flash-lite") ensemble/3-model-blend → ("ensemble", None, "3-model-blend") - synthpanel/claude-haiku-4-5-20251001 → ("synthpanel", "anthropic", "claude-haiku-4-5-20251001") - synthpanel/gemini-2.5-flash-lite → ("synthpanel", "google", "gemini-2.5-flash-lite") + althing/claude-haiku-4-5-20251001 → ("althing", "anthropic", "claude-haiku-4-5-20251001") + althing/gemini-2.5-flash-lite → ("althing", "google", "gemini-2.5-flash-lite") openrouter/openai/gpt-4o-mini → ("raw", "openai", "gpt-4o-mini") - synthpanel/openrouter/anthropic/claude-haiku-4-5 - → ("synthpanel", "anthropic", "claude-haiku-4-5") + althing/openrouter/anthropic/claude-haiku-4-5 + → ("althing", "anthropic", "claude-haiku-4-5") OpenRouter is a gateway, not a framework — paths that lead with ``openrouter/`` collapse to ``framework=raw`` so the same model reached @@ -217,8 +217,8 @@ def _runnable_model_slug(head: str) -> str: openrouter/google/gemini-2.5-flash-lite → google/gemini-2.5-flash-lite openrouter/meta-llama/llama-3.3-70b-instruct → meta-llama/llama-3.3-70b-instruct - synthpanel/openrouter/anthropic/claude-haiku-4-5 → anthropic/claude-haiku-4-5 - synthpanel/gemini-2.5-flash-lite → google/gemini-2.5-flash-lite + althing/openrouter/anthropic/claude-haiku-4-5 → anthropic/claude-haiku-4-5 + althing/gemini-2.5-flash-lite → google/gemini-2.5-flash-lite raw-anthropic/claude-haiku-4-5-20251001 → anthropic/claude-haiku-4-5-20251001 raw-gemini/gemini-2.5-flash-lite → google/gemini-2.5-flash-lite ensemble/3-model-blend → 3-model-blend @@ -232,7 +232,9 @@ def _runnable_model_slug(head: str) -> str: parts = head.split("/") # Drop a leading runner segment that isn't part of the model id. - if parts and parts[0] in ("synthpanel", "ensemble"): + # ("synthpanel" is the pre-rename id of the althing runner, present in + # historical config ids.) + if parts and parts[0] in ("althing", "synthpanel", "ensemble"): parts = parts[1:] # Drop an OpenRouter gateway hop — the remaining segments ARE the slug. if parts and parts[0] == "openrouter": @@ -258,9 +260,9 @@ def runnable_ids(provider: str) -> tuple[str, str]: These are the runnable counterparts to the human-facing display labels: - ``provider_id`` — the execution framework/runner taxonomy from - :func:`parse_provider` (``raw`` / ``synthpanel`` / ``ensemble`` / + :func:`parse_provider` (``raw`` / ``althing`` / ``ensemble`` / ``baseline``). Tells a consumer whether a score reflects a raw model, the - SynthPanel product layer, an ensemble, or a statistical baseline. + Althing product layer, an ensemble, or a statistical baseline. - ``model_id`` — an OpenRouter-runnable ``/`` slug a consumer can pipe straight to the gateway without parsing display names. See :func:`_runnable_model_slug` for the exact derivation per shape. diff --git a/src/synthbench/convergence/baseline.py b/src/synthbench/convergence/baseline.py index 0b7766fb..861beec8 100644 --- a/src/synthbench/convergence/baseline.py +++ b/src/synthbench/convergence/baseline.py @@ -1,6 +1,6 @@ -"""Stable baseline-loader export for synthpanel's ``--calibrate-against`` flag. +"""Stable baseline-loader export for althing's ``--calibrate-against`` flag. -synthpanel (>= 0.9) probes ``synthbench`` at runtime for a +althing (>= 0.9) probes ``synthbench`` at runtime for a ``load_convergence_baseline`` callable when a user passes ``--calibrate-against DATASET:QUESTION``. This module is the single owner of that export. @@ -20,13 +20,13 @@ lookup is conservative (defaults to ``aggregates_only``) and a typo should not silently leak or crash downstream. -The return shape is fixed by ``docs/convergence-analysis.md`` — synthpanel +The return shape is fixed by ``docs/convergence-analysis.md`` — althing attaches ``human_distribution`` to every matching question's ``calibration`` sub-object and computes JSD with its own local metric. For datasets whose policy permits redistributing the question wording (``full`` / ``citation_only``), the payload additionally carries -``question_text`` and ``options`` so synthpanel can present the *real* +``question_text`` and ``options`` so althing can present the *real* survey item to its panel instead of fabricating a placeholder prompt. These fields are additive; ``human_distribution`` remains the only load-bearing key. @@ -52,7 +52,7 @@ class BaselineGatedError(BaselineUnavailable): """Raised when the dataset is ``gated`` and requires authenticated access. The in-process loader cannot vend gated-tier baselines — those payloads - live behind the JWT-authenticated R2 origin. synthpanel can surface this + live behind the JWT-authenticated R2 origin. althing can surface this to the user as an "authenticate to calibrate against " hint. """ @@ -133,7 +133,7 @@ def load_convergence_baseline( } # Additive, policy-gated metadata: the survey item's wording and its - # answer labels. synthpanel's calibration panel needs the real question + # answer labels. althing's calibration panel needs the real question # text (rather than a fabricated placeholder) and needs option labels that # line up with the ``human_distribution`` keys so its pick_one extraction # aligns with ground truth. Only surfaced where the redistribution policy @@ -150,7 +150,7 @@ def _resolve_question(dataset, question_key: str): """Locate a Question by key; accept bare or adapter-prefixed forms. Mirrors the CLI's :func:`convergence.cli_report._resolve_question` - behavior so synthpanel callers can pass either the canonical + behavior so althing callers can pass either the canonical ``Question.key`` or the bare upstream id. """ questions = dataset.load() diff --git a/src/synthbench/findings.py b/src/synthbench/findings.py index 546bc763..c9aec714 100644 --- a/src/synthbench/findings.py +++ b/src/synthbench/findings.py @@ -31,18 +31,18 @@ """ # Display-name map: product entries -> underlying raw-LLM display name. -# Also used by publish._annotate_normalized_sps so a SynthPanel row can look +# Also used by publish._annotate_normalized_sps so a Althing row can look # up its corresponding "just prompt the model" baseline SPS for the same # dataset. Kept as a small explicit map (rather than string parsing) because # the product display names omit the vendor prefix ("Haiku 4.5" vs "Claude # Haiku 4.5") and a typo-tolerant match would silently bind the wrong row. PRODUCT_TO_RAW_DISPLAY: dict[str, str] = { - "SynthPanel (Haiku 4.5)": "Claude Haiku 4.5", - "SynthPanel (Sonnet 4)": "Claude Sonnet 4", - "SynthPanel (GPT-4o-mini)": "GPT-4o-mini", - "SynthPanel (GPT-4o)": "GPT-4o", - "SynthPanel (Gemini Flash Lite)": "Gemini Flash Lite", - "SynthPanel (Llama 3.3 70B)": "Llama 3.3 70B", + "Althing (Haiku 4.5)": "Claude Haiku 4.5", + "Althing (Sonnet 4)": "Claude Sonnet 4", + "Althing (GPT-4o-mini)": "GPT-4o-mini", + "Althing (GPT-4o)": "GPT-4o", + "Althing (Gemini Flash Lite)": "Gemini Flash Lite", + "Althing (Llama 3.3 70B)": "Llama 3.3 70B", } # Short display labels for demographic groups whose raw SubPOP labels are @@ -937,7 +937,7 @@ def build_findings( "ensemble_signed_error_correlation asserted constant." ), "temperature_sweep": ( - "OpinionsQA sweep-tagged SynthPanel runs (' t=' provider suffix); " + "OpinionsQA sweep-tagged Althing runs (' t=' provider suffix); " "mean/std of recomputed SPS across the n_runs replications per " "(model, temperature) cell." ), diff --git a/src/synthbench/leaderboard.py b/src/synthbench/leaderboard.py index 1ac66736..10badf89 100644 --- a/src/synthbench/leaderboard.py +++ b/src/synthbench/leaderboard.py @@ -27,46 +27,57 @@ "raw-anthropic/claude-haiku-4-5-20251001": ("Claude Haiku 4.5", "raw"), "raw-gemini/gemini-2.5-flash-lite": ("Gemini Flash Lite", "raw"), # Products (orchestration / adapter layers) - "synthpanel/claude-haiku-4-5-20251001": ("SynthPanel (Haiku 4.5)", "product"), - "synthpanel/gemini-2.5-flash-lite": ( - "SynthPanel (Gemini Flash Lite)", + "althing/claude-haiku-4-5-20251001": ("Althing (Haiku 4.5)", "product"), + "althing/gemini-2.5-flash-lite": ( + "Althing (Gemini Flash Lite)", "product", ), # Products — via OpenRouter - "synthpanel/openrouter/anthropic/claude-haiku-4-5": ( - "SynthPanel (Haiku 4.5)", + "althing/openrouter/anthropic/claude-haiku-4-5": ( + "Althing (Haiku 4.5)", "product", ), - "synthpanel/openrouter/google/gemini-2.5-flash-lite": ( - "SynthPanel (Gemini Flash Lite)", + "althing/openrouter/google/gemini-2.5-flash-lite": ( + "Althing (Gemini Flash Lite)", "product", ), - "synthpanel/openrouter/openai/gpt-4o-mini": ( - "SynthPanel (GPT-4o-mini)", + "althing/openrouter/openai/gpt-4o-mini": ( + "Althing (GPT-4o-mini)", "product", ), - "synthpanel/openrouter/anthropic/claude-sonnet-4": ( - "SynthPanel (Sonnet 4)", + "althing/openrouter/anthropic/claude-sonnet-4": ( + "Althing (Sonnet 4)", "product", ), - "synthpanel/openrouter/meta-llama/llama-3.3-70b-instruct": ( - "SynthPanel (Llama 3.3 70B)", + "althing/openrouter/meta-llama/llama-3.3-70b-instruct": ( + "Althing (Llama 3.3 70B)", "product", ), - "synthpanel/gpt-4o-mini": ("SynthPanel (GPT-4o-mini)", "product"), + "althing/gpt-4o-mini": ("Althing (GPT-4o-mini)", "product"), # Ensemble - "ensemble/3-model-blend": ("SynthPanel Ensemble (3-model)", "product"), + "ensemble/3-model-blend": ("Althing Ensemble (3-model)", "product"), # Baselines "random-baseline": ("Random Baseline", "baseline"), "majority-baseline": ("Majority Baseline", "baseline"), } +# Historical alias: results recorded before the 2026-07 rename carry the +# `synthpanel/` provider prefix. Same taxonomy entry, old id — display names +# intentionally show the current brand (Althing) for both. +MODEL_MAP.update( + { + key.replace("althing/", "synthpanel/", 1): value + for key, value in list(MODEL_MAP.items()) + if key.startswith("althing/") + } +) + def _parse_provider_base(provider: str) -> str: """Strip temperature/template suffixes from provider name. - 'synthpanel/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current' - → 'synthpanel/openrouter/anthropic/claude-haiku-4-5' + 'althing/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current' + → 'althing/openrouter/anthropic/claude-haiku-4-5' """ return provider.split(" t=")[0].split(" tpl=")[0].split(" profile=")[0].strip() @@ -113,8 +124,8 @@ def _config_key(result: dict) -> tuple: def _canonicalize_provider_model_segment(provider: str) -> str: """Strip a version-date suffix from the model (last) path segment. - ``synthpanel/claude-haiku-4-5-20251001`` → - ``synthpanel/claude-haiku-4-5``. Used to collapse dated and undated + ``althing/claude-haiku-4-5-20251001`` → + ``althing/claude-haiku-4-5``. Used to collapse dated and undated aliases of the same underlying model when looking up MODEL_MAP. """ from synthbench.config_id import canonical_model @@ -132,7 +143,7 @@ def _canonicalize_provider_model_segment(provider: str) -> str: def display_provider_name(provider: str) -> str: """Map full provider paths to human-friendly display names via MODEL_MAP. - Handles suffixed names like 'synthpanel/... t=0.85 tpl=current' by + Handles suffixed names like 'althing/... t=0.85 tpl=current' by stripping suffixes before lookup. Also normalizes Anthropic-style ``-YYYYMMDD`` and OpenAI-style ``-YYYY-MM-DD`` model-version dates so aliased + dated forms collapse to a single display name. diff --git a/src/synthbench/providers/__init__.py b/src/synthbench/providers/__init__.py index 82c76b63..3b9568d3 100644 --- a/src/synthbench/providers/__init__.py +++ b/src/synthbench/providers/__init__.py @@ -12,7 +12,11 @@ "raw-gemini": "synthbench.providers.raw_gemini:RawGeminiProvider", "openrouter": "synthbench.providers.openrouter:OpenRouterProvider", "ollama": "synthbench.providers.ollama:OllamaProvider", - "synthpanel": "synthbench.providers.synthpanel:SynthPanelProvider", + "althing": "synthbench.providers.althing:AlthingProvider", + # Deprecated alias — synthpanel was renamed to althing (2026-07). Kept so + # existing configs and `--provider synthpanel` invocations keep working; + # leaderboard-results/ entries recorded under the old id remain valid. + "synthpanel": "synthbench.providers.althing:AlthingProvider", "http": "synthbench.providers.http:HttpProvider", "random": "synthbench.providers.random_baseline:RandomBaselineProvider", "majority": "synthbench.providers.majority_baseline:MajorityBaselineProvider", diff --git a/src/synthbench/providers/synthpanel.py b/src/synthbench/providers/althing.py similarity index 92% rename from src/synthbench/providers/synthpanel.py rename to src/synthbench/providers/althing.py index 7660330e..c1c76fbe 100644 --- a/src/synthbench/providers/synthpanel.py +++ b/src/synthbench/providers/althing.py @@ -1,8 +1,8 @@ -"""SynthPanel full-pipeline provider. +"""Althing full-pipeline provider. -Benchmarks the complete SynthPanel pipeline. Prefers direct Python API +Benchmarks the complete Althing pipeline. Prefers direct Python API import when available (zero subprocess overhead); falls back to the -``synthpanel`` CLI. +``althing`` CLI. """ from __future__ import annotations @@ -29,11 +29,11 @@ _LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" # --------------------------------------------------------------------------- -# Optional direct API import (requires synth_panel on Python 3.10+) +# Optional direct API import (requires althing on Python 3.10+) # --------------------------------------------------------------------------- try: - from synth_panel.llm.client import LLMClient - from synth_panel.llm.models import ( + from althing.llm.client import LLMClient + from althing.llm.models import ( CompletionRequest, InputMessage, TextBlock, @@ -43,7 +43,20 @@ _HAS_SYNTH_PANEL_API = True except (ImportError, TypeError): - _HAS_SYNTH_PANEL_API = False + try: + # Pre-rename installs expose the same API as synth_panel. + from synth_panel.llm.client import LLMClient # noqa: F401 + from synth_panel.llm.models import ( # noqa: F401 + CompletionRequest, + InputMessage, + TextBlock, + ToolChoice, + ToolDefinition, + ) + + _HAS_SYNTH_PANEL_API = True + except (ImportError, TypeError): + _HAS_SYNTH_PANEL_API = False def _pick_raw_sample(samples: list[tuple[str, Any]], counts: Counter) -> dict | None: @@ -73,7 +86,7 @@ def _yaml_escape(text: str) -> str: def _build_instrument_yaml(question: str, options: list[str]) -> str: """Build a single-question instrument YAML with natural presentation. - Embeds the options in the question text so synthpanel sees a natural + Embeds the options in the question text so althing sees a natural survey question rather than a forced-choice letter prompt. """ opts_lines = "\\n".join(f"({_LETTERS[i]}) {opt}" for i, opt in enumerate(options)) @@ -95,7 +108,7 @@ def _build_multi_question_instrument_yaml( ) -> str: """Build a multi-question instrument YAML for batch evaluation. - Packs N questions into a single instrument so synthpanel processes + Packs N questions into a single instrument so althing processes them all in one invocation. """ lines = [ @@ -118,7 +131,7 @@ def _build_persona_yaml(persona: PersonaSpec | None, count: int = 1) -> str: """Build persona YAML with full conditioning context. When *count* > 1, replicates the persona to enable batch runs - through a single synthpanel invocation. + through a single althing invocation. """ if persona is None: block = ( @@ -160,7 +173,7 @@ def _build_persona_yaml(persona: PersonaSpec | None, count: int = 1) -> str: def _build_system_prompt(persona: PersonaSpec | None) -> str: - """Build a system prompt matching synthpanel's ``persona_system_prompt``.""" + """Build a system prompt matching althing's ``persona_system_prompt``.""" if persona is None: return ( "You are role-playing as Survey Respondent. " @@ -257,14 +270,14 @@ def _parse_structured_response(response: Any, options: list[str]): return getattr(response, "text", "") or "", ParsedResponse() -class SynthPanelProvider(Provider): - """Benchmark the full SynthPanel pipeline. +class AlthingProvider(Provider): + """Benchmark the full Althing pipeline. - When ``synth_panel`` is importable, uses the Python API directly — + When ``althing`` is importable, uses the Python API directly — no subprocess overhead (~1s saved per call). Otherwise falls back - to ``synthpanel panel run`` via subprocess. + to ``althing panel run`` via subprocess. - Supports synthpanel v0.6.0+ flags: --models, --temperature, --profile. + Supports althing v0.6.0+ flags: --models, --temperature, --profile. """ def __init__( @@ -273,7 +286,7 @@ def __init__( temperature: float | None = None, profile: str | None = None, prompt_template: str | None = None, - synthpanel_path: str | None = None, + althing_path: str | None = None, elicitation: str = "natural", ): if elicitation not in ELICITATION_MODES: @@ -287,9 +300,9 @@ def __init__( ) if elicitation == "structured" and not _HAS_SYNTH_PANEL_API: raise ImportError( - "elicitation='structured' requires the synth_panel Python API " - "(schema-forced tool calls are not exposed by the synthpanel " - "CLI fallback). Install synth_panel on Python 3.10+." + "elicitation='structured' requires the althing Python API " + "(schema-forced tool calls are not exposed by the althing " + "CLI fallback). Install althing on Python 3.10+." ) self._model = model self._temperature = temperature @@ -297,7 +310,7 @@ def __init__( self._prompt_template = prompt_template self._elicitation = elicitation # The direct API path builds its own persona/system prompt and has - # no seam for a synthpanel --prompt-template override; only the CLI + # no seam for a althing --prompt-template override; only the CLI # honours it. Route template-override runs through the CLI so the # knob is actually applied — a row tagged ``tpl=`` whose # requests never saw the template would be untrue leaderboard @@ -310,20 +323,22 @@ def __init__( self._client = LLMClient() self._executor = ThreadPoolExecutor(max_workers=16) else: - if synthpanel_path: - self._synthpanel_bin = synthpanel_path + if althing_path: + self._althing_bin = althing_path else: - found = shutil.which("synthpanel") + # Prefer the new CLI name; fall back to the pre-rename + # `synthpanel` binary for installs older than the rename. + found = shutil.which("althing") or shutil.which("synthpanel") if found is None: raise ImportError( - "synthpanel is not installed or not on PATH. " - "Install synthpanel or pass synthpanel_path= explicitly." + "althing is not installed or not on PATH. " + "Install althing or pass althing_path= explicitly." ) - self._synthpanel_bin = found + self._althing_bin = found @property def name(self) -> str: - parts = [f"synthpanel/{self._model}"] + parts = [f"althing/{self._model}"] if self._temperature is not None: parts.append(f"t={self._temperature}") if self._profile: @@ -548,7 +563,7 @@ def _one_call(): if infra_errors and not responses and not refusals and not parse_failures: raise ProviderError( - f"synthpanel API: all {len(infra_errors)} samples failed with " + f"althing API: all {len(infra_errors)} samples failed with " f"infrastructure errors (last: {infra_errors[-1]!r})" ) from infra_errors[-1] @@ -588,7 +603,7 @@ async def _get_distribution_cli( persona: PersonaSpec | None, n_samples: int | None, ) -> Distribution: - """Batch N identical personas into one synthpanel invocation.""" + """Batch N identical personas into one althing invocation.""" effective_samples = n_samples if n_samples is not None else 30 instrument_yaml = _build_instrument_yaml(question, options) persona_yaml = _build_persona_yaml(persona, count=effective_samples) @@ -625,10 +640,10 @@ async def batch_respond( *, persona: PersonaSpec | None = None, ) -> list[Response]: - """Answer multiple questions in a single synthpanel invocation. + """Answer multiple questions in a single althing invocation. Packs all questions into one multi-question instrument, runs - synthpanel once, and returns per-question Response objects. + althing once, and returns per-question Response objects. """ if self._use_api: return await self._batch_respond_api(questions, options_list, persona) @@ -691,7 +706,7 @@ async def batch_get_distribution( """Get distributions for multiple questions in a single invocation. CLI path: packs questions into one multi-question instrument with - N personas, runs synthpanel once, and extracts per-question + N personas, runs althing once, and extracts per-question distributions from the results. API path: gathers individual get_distribution calls concurrently. @@ -756,9 +771,9 @@ async def _batch_get_distribution_cli( # ================================================================== def _build_cmd(self, inst_path: str, pers_path: str) -> list[str]: - """Build the synthpanel CLI command (no --no-synthesis).""" + """Build the althing CLI command (no --no-synthesis).""" cmd = [ - self._synthpanel_bin, + self._althing_bin, "--output-format", "json", ] @@ -783,7 +798,7 @@ def _build_cmd(self, inst_path: str, pers_path: str) -> list[str]: return cmd async def _run_subprocess(self, cmd: list[str]) -> dict: - """Run the synthpanel CLI and return its parsed JSON output. + """Run the althing CLI and return its parsed JSON output. A non-zero exit or unparseable output is an infrastructure failure of the pipeline under benchmark — raise instead of fabricating a @@ -800,21 +815,19 @@ async def _run_subprocess(self, cmd: list[str]) -> dict: raw_stderr = stderr.decode().strip() if proc.returncode != 0: - raise ProviderError( - f"synthpanel exited {proc.returncode}: {raw_stderr[:500]}" - ) + raise ProviderError(f"althing exited {proc.returncode}: {raw_stderr[:500]}") try: return json.loads(raw_stdout) except json.JSONDecodeError as exc: raise ProviderError( - f"failed to parse synthpanel JSON output: {raw_stdout[:200]!r}" + f"failed to parse althing JSON output: {raw_stdout[:200]!r}" ) from exc async def _run_cli( self, inst_path: str, pers_path: str, options: list[str] ) -> Response: - """Execute synthpanel CLI and parse the JSON output.""" + """Execute althing CLI and parse the JSON output.""" cmd = self._build_cmd(inst_path, pers_path) data = await self._run_subprocess(cmd) @@ -827,7 +840,7 @@ async def _run_cli( parsed = parse_option_response(raw_text, options) - # Gather metadata from synthpanel output + # Gather metadata from althing output panelist_result = {} try: panelist_result = data["rounds"][0]["results"][0] @@ -902,7 +915,7 @@ async def _run_batch( async def _run_multi_cli( self, inst_path: str, pers_path: str, options_list: list[list[str]] ) -> list[Response]: - """Execute synthpanel with a multi-question instrument (single persona).""" + """Execute althing with a multi-question instrument (single persona).""" cmd = self._build_cmd(inst_path, pers_path) data = await self._run_subprocess(cmd) @@ -1010,3 +1023,7 @@ async def _run_multi_batch( async def close(self) -> None: if self._executor is not None: self._executor.shutdown(wait=False) + + +# Deprecated alias — kept for external imports that predate the rename. +SynthPanelProvider = AlthingProvider diff --git a/src/synthbench/publish.py b/src/synthbench/publish.py index f02c6877..b595c43d 100644 --- a/src/synthbench/publish.py +++ b/src/synthbench/publish.py @@ -399,7 +399,7 @@ def _estimate_cost_fields(config: dict, entry: dict) -> dict: if not provider: return dict(_NULL_COST_FIELDS) try: - from synth_panel.cost import lookup_pricing_by_provider + from althing.cost import lookup_pricing_by_provider except ImportError: return dict(_NULL_COST_FIELDS) pricing, _ = lookup_pricing_by_provider(provider) @@ -469,7 +469,7 @@ def _compute_cost_fields(aggregate: dict, config: dict, entry: dict) -> dict: if not provider: return dict(_NULL_COST_FIELDS) try: - from synth_panel.cost import lookup_pricing_by_provider + from althing.cost import lookup_pricing_by_provider except ImportError: return dict(_NULL_COST_FIELDS) pricing, _is_estimated = lookup_pricing_by_provider(provider) @@ -526,7 +526,7 @@ def _compute_ensemble_cost( return None try: - from synth_panel.cost import lookup_pricing_by_provider + from althing.cost import lookup_pricing_by_provider except ImportError: return None @@ -567,16 +567,16 @@ def _compute_ensemble_cost( def _build_pricing_snapshot() -> dict: """Emit the rates table used for this publish run. - Reads the named pricing constants from ``synth_panel.cost`` plus the + Reads the named pricing constants from ``althing.cost`` plus the ``# pricing snapshot_date: YYYY-MM-DD`` anchor comment, and stamps the - installed synthpanel version. Designed to be lossless under ``json.dump``. + installed althing version. Designed to be lossless under ``json.dump``. """ rates: dict[str, dict] = {} snapshot_date: str | None = None - synth_panel_version: str | None = None + althing_version: str | None = None try: - from synth_panel import cost as _spc + from althing import cost as _spc except ImportError: _spc = None # type: ignore[assignment] @@ -613,15 +613,15 @@ def _build_pricing_snapshot() -> dict: from importlib.metadata import PackageNotFoundError, version try: - synth_panel_version = version("synthpanel") + althing_version = version("althing") except PackageNotFoundError: - synth_panel_version = None + althing_version = None except ImportError: - synth_panel_version = None + althing_version = None return { "generated_at": datetime.now(timezone.utc).isoformat(), - "synth_panel_version": synth_panel_version, + "althing_version": althing_version, "snapshot_date": snapshot_date, "rates": rates, } @@ -1279,7 +1279,7 @@ def publish_leaderboard_data( # Index by (provider_string, dataset) so ensemble cost can resolve its # constituent runs. Built off the deduped pool — ensemble_sources reference - # canonical provider strings (e.g., "synthpanel/claude-haiku-4-5-...") that + # canonical provider strings (e.g., "althing/claude-haiku-4-5-...") that # match the constituent runs surviving dedup. results_by_provider_ds: dict[tuple[str, str], dict] = {} for r in deduped: diff --git a/src/synthbench/run_validity.py b/src/synthbench/run_validity.py index be96aacb..3792a3f5 100644 --- a/src/synthbench/run_validity.py +++ b/src/synthbench/run_validity.py @@ -4,7 +4,7 @@ overwhelmingly perfectly uniform (``{A: 0.25, B: 0.25, ...}`` for a 4-option question), coupled with near-zero refusal activity. This is the signature of a silent API failure — provider returned empty distributions, model alias -missed, budget exhaustion, synthpanel fallback — that still parsed cleanly +missed, budget exhaustion, althing fallback — that still parsed cleanly but produced no usable signal. Existing ``parse_failure_rate`` tracking does NOT catch this class: the run diff --git a/src/synthbench/runner.py b/src/synthbench/runner.py index bbfcbcf8..6ed0628d 100644 --- a/src/synthbench/runner.py +++ b/src/synthbench/runner.py @@ -589,7 +589,7 @@ def _build_question_result( token_usage = dist.metadata.get("usage") if dist.metadata else None # Same raw_sample extraction as _evaluate_question — the batched # path previously dropped provider-attached samples on the floor, - # which left batch-capable distribution providers (synthpanel) + # which left batch-capable distribution providers (althing) # unable to populate raw_responses at all. raw_sample: dict | None = None if dist.metadata: diff --git a/src/synthbench/stats.py b/src/synthbench/stats.py index bd25aeaa..8819fe61 100644 --- a/src/synthbench/stats.py +++ b/src/synthbench/stats.py @@ -1,4 +1,4 @@ -# Vendored from synthpanel (DataViking-Tech/SynthPanel). Keep in sync. +# Vendored from althing (DataViking-Tech/Althing). Keep in sync. """Pure-Python statistical tests for synthetic panel analysis. All functions use only ``math``, ``random``, and ``dataclasses`` from stdlib. diff --git a/templates/user-configs/example-healthcare-northcentral.yaml b/templates/user-configs/example-healthcare-northcentral.yaml index 3b16e227..0b77caf9 100644 --- a/templates/user-configs/example-healthcare-northcentral.yaml +++ b/templates/user-configs/example-healthcare-northcentral.yaml @@ -1,4 +1,4 @@ -# Example user-config: synthpanel tuned for Midwest healthcare workers. +# Example user-config: althing tuned for Midwest healthcare workers. # See docs/configs.md for the full schema. # # Validate locally: @@ -8,7 +8,7 @@ # synthbench submit --config templates/user-configs/example-healthcare-northcentral.yaml \ # results/my-run.json -vendor: synthpanel +vendor: althing vendor_version: "1.4.0" config_name: "example-healthcare-northcentral" contributor: "@wesley" diff --git a/tests/test_config_id.py b/tests/test_config_id.py index 724aa0a0..9ba59cab 100644 --- a/tests/test_config_id.py +++ b/tests/test_config_id.py @@ -34,23 +34,23 @@ def test_openrouter_meta_llama_normalizes_to_meta(self): assert p.base_provider == "meta" assert p.model == "llama-3.3-70b-instruct" - def test_synthpanel_nested_four_parts(self): - p = parse_provider("synthpanel/openrouter/anthropic/claude-haiku-4-5") - assert p.framework == "synthpanel" + def test_althing_nested_four_parts(self): + p = parse_provider("althing/openrouter/anthropic/claude-haiku-4-5") + assert p.framework == "althing" assert p.base_provider == "anthropic" assert p.model == "claude-haiku-4-5" assert p.knobs == {} - def test_synthpanel_direct_claude_infers_anthropic(self): - """`synthpanel/` with no vendor segment infers from model name.""" - p = parse_provider("synthpanel/claude-haiku-4-5-20251001") - assert p.framework == "synthpanel" + def test_althing_direct_claude_infers_anthropic(self): + """`althing/` with no vendor segment infers from model name.""" + p = parse_provider("althing/claude-haiku-4-5-20251001") + assert p.framework == "althing" assert p.base_provider == "anthropic" assert p.model == "claude-haiku-4-5-20251001" - def test_synthpanel_direct_gemini_infers_google(self): - p = parse_provider("synthpanel/gemini-2.5-flash-lite") - assert p.framework == "synthpanel" + def test_althing_direct_gemini_infers_google(self): + p = parse_provider("althing/gemini-2.5-flash-lite") + assert p.framework == "althing" assert p.base_provider == "google" assert p.model == "gemini-2.5-flash-lite" @@ -87,17 +87,17 @@ def test_majority_baseline(self): def test_knobs_parsed(self): p = parse_provider( - "synthpanel/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current" + "althing/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current" ) assert p.knobs == {"t": "0.85", "tpl": "current"} def test_knobs_order_independent(self): - a = parse_provider("synthpanel/x/y/z t=0.5 tpl=minimal") - b = parse_provider("synthpanel/x/y/z tpl=minimal t=0.5") + a = parse_provider("althing/x/y/z t=0.5 tpl=minimal") + b = parse_provider("althing/x/y/z tpl=minimal t=0.5") assert a.knobs == b.knobs def test_malformed_knob_ignored(self): - p = parse_provider("synthpanel/x/y/z notakey t=0.5") + p = parse_provider("althing/x/y/z notakey t=0.5") assert p.knobs == {"t": "0.5"} def test_empty_string(self): @@ -114,7 +114,7 @@ def test_extra_whitespace_tolerated(self): class TestBuildConfigId: def test_slug_shape(self): slug, _ = build_config_id( - "synthpanel/openrouter/anthropic/claude-haiku-4-5", + "althing/openrouter/anthropic/claude-haiku-4-5", dataset="opinionsqa", temperature=0.85, template="current", @@ -122,7 +122,7 @@ def test_slug_shape(self): # framework--model--t--tpl-- parts = slug.split("--") assert len(parts) == 5 - assert parts[0] == "synthpanel" + assert parts[0] == "althing" assert parts[1] == "claude-haiku-4-5" assert parts[2] == "t0.85" assert parts[3] == "tplcurrent" @@ -131,7 +131,7 @@ def test_slug_shape(self): def test_hash_deterministic(self): args = { - "provider": "synthpanel/openrouter/openai/gpt-4o-mini", + "provider": "althing/openrouter/openai/gpt-4o-mini", "dataset": "opinionsqa", "temperature": 0.5, "template": "current", @@ -144,11 +144,11 @@ def test_knob_order_does_not_affect_hash(self): """Same canonical config from different provider-string orderings must produce the same slug.""" s1, _ = build_config_id( - "synthpanel/x/y/z t=0.85 tpl=current", + "althing/x/y/z t=0.85 tpl=current", dataset="opinionsqa", ) s2, _ = build_config_id( - "synthpanel/x/y/z tpl=current t=0.85", + "althing/x/y/z tpl=current t=0.85", dataset="opinionsqa", ) assert s1 == s2 @@ -157,14 +157,14 @@ def test_collision_resistance_on_samples(self): """Two runs identical in framework/model/temp/template but differing in samples_per_question must produce different config IDs.""" s1, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="current", samples_per_question=30, ) s2, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="current", @@ -176,14 +176,14 @@ def test_collision_resistance_on_samples(self): def test_collision_resistance_on_question_set(self): s1, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="current", question_set_hash="abc123", ) s2, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="current", @@ -193,13 +193,13 @@ def test_collision_resistance_on_question_set(self): def test_collision_resistance_on_dataset(self): s_oqa, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="current", ) s_sub, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="subpop", temperature=0.85, template="current", @@ -221,7 +221,7 @@ def test_explicit_temperature_overrides_knob(self): """The real config.temperature should win over knob ``t=`` in the provider string — the knob is advisory and may lag the actual run.""" slug, parsed = build_config_id( - "synthpanel/x/y/z t=0.3", + "althing/x/y/z t=0.3", dataset="opinionsqa", temperature=0.85, template="current", @@ -232,7 +232,7 @@ def test_explicit_temperature_overrides_knob(self): def test_template_path_stripped(self): """Template stored as a file path should have extension/dir stripped.""" slug, _ = build_config_id( - "synthpanel/x/y/z", + "althing/x/y/z", dataset="opinionsqa", temperature=0.85, template="templates/minimal.md", @@ -321,17 +321,17 @@ class TestBuildConfigIdCanonicalization: """config_id collapses dated + undated aliases of the same model.""" def test_anthropic_dated_and_aliased_produce_same_slug(self): - """`synthpanel/claude-haiku-4-5-20251001` and - `synthpanel/openrouter/anthropic/claude-haiku-4-5` are the same + """`althing/claude-haiku-4-5-20251001` and + `althing/openrouter/anthropic/claude-haiku-4-5` are the same underlying model — their config_ids must collapse.""" s_dated, p_dated = build_config_id( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", dataset="opinionsqa", temperature=0.85, template="current", ) s_alias, p_alias = build_config_id( - "synthpanel/openrouter/anthropic/claude-haiku-4-5", + "althing/openrouter/anthropic/claude-haiku-4-5", dataset="opinionsqa", temperature=0.85, template="current", @@ -377,12 +377,12 @@ def test_dated_and_aliased_collide_across_datasets(self): """Canonicalization applies regardless of dataset — but different datasets still produce different hashes (no cross-dataset collapse).""" s1, _ = build_config_id( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", dataset="opinionsqa", temperature=0.85, ) s2, _ = build_config_id( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", dataset="subpop", temperature=0.85, ) diff --git a/tests/test_convergence_baseline.py b/tests/test_convergence_baseline.py index 742ad273..441f3340 100644 --- a/tests/test_convergence_baseline.py +++ b/tests/test_convergence_baseline.py @@ -1,11 +1,11 @@ """Tests for ``synthbench.load_convergence_baseline`` (sb-ham8). Contract defined in ``docs/convergence-analysis.md`` §"Integration: -synthpanel --calibrate-against". synthpanel (>= 0.9) probes six attribute +althing --calibrate-against". althing (>= 0.9) probes six attribute paths to find a baseline loader; the top-of-list path is ``synthbench.load_convergence_baseline(dataset=..., question_key=...)``. -These tests pin the behavior synthpanel expects: +These tests pin the behavior althing expects: * The export is visible both at ``synthbench.*`` and ``synthbench.convergence.*`` (it probes both namespaces). @@ -34,18 +34,18 @@ # --------------------------------------------------------------------------- -# synthpanel attribute probe — the export must resolve at both paths +# althing attribute probe — the export must resolve at both paths # --------------------------------------------------------------------------- def test_export_visible_at_module_root(): - """synthpanel's first probe hits ``synthbench.load_convergence_baseline``.""" + """althing's first probe hits ``synthbench.load_convergence_baseline``.""" assert hasattr(synthbench, "load_convergence_baseline") assert callable(synthbench.load_convergence_baseline) def test_export_visible_under_convergence(): - """synthpanel's fallback probe hits ``synthbench.convergence.*``.""" + """althing's fallback probe hits ``synthbench.convergence.*``.""" assert lcb_convergence is load_convergence_baseline @@ -110,10 +110,10 @@ def test_full_tier_gss_returns_documented_payload(gss_with_data): assert payload["dataset"] == "gss" assert payload["question_key"] == "GSS_SPKATH" assert payload["redistribution_policy"] == "full" - # human_distribution is the only load-bearing field — synthpanel attaches + # human_distribution is the only load-bearing field — althing attaches # it verbatim to each matching question's `calibration` sub-object. assert payload["human_distribution"] == pytest.approx({"Yes": 0.7, "No": 0.3}) - # Provenance passes through for synthpanel's run-metadata block. + # Provenance passes through for althing's run-metadata block. assert payload["license_url"] assert payload["citation"] @@ -121,7 +121,7 @@ def test_full_tier_gss_returns_documented_payload(gss_with_data): def test_full_tier_gss_includes_question_text_and_options(gss_with_data): """sb-qtext: full-tier payloads carry the real question wording + labels. - synthpanel's calibration panel must present the *real* survey item rather + althing's calibration panel must present the *real* survey item rather than fabricate a placeholder, and its pick_one labels must line up with the ``human_distribution`` keys. """ @@ -129,7 +129,7 @@ def test_full_tier_gss_includes_question_text_and_options(gss_with_data): assert payload["question_text"] == "Allow atheist to speak?" assert payload["options"] == ["Yes", "No"] # Options are aligned with the human_distribution keys (same strings), so - # synthpanel's derived pick_one schema maps 1:1 onto ground truth. + # althing's derived pick_one schema maps 1:1 onto ground truth. assert set(payload["options"]) == set(payload["human_distribution"].keys()) @@ -173,7 +173,7 @@ def test_full_tier_ntia_includes_question_text_and_options(monkeypatch): def test_full_tier_accepts_bare_question_id(gss_with_data): - """synthpanel may pass the upstream id (``SPKATH``) without the + """althing may pass the upstream id (``SPKATH``) without the ``GSS_`` prefix — the loader resolves either form.""" payload = load_convergence_baseline(dataset="gss", question_key="SPKATH") assert payload["question_key"] == "GSS_SPKATH" @@ -193,7 +193,7 @@ def test_unknown_question_raises_unavailable(gss_with_data): # --------------------------------------------------------------------------- -# Gated failure path — synthpanel must see a distinct, catchable error +# Gated failure path — althing must see a distinct, catchable error # --------------------------------------------------------------------------- diff --git a/tests/test_effort.py b/tests/test_effort.py index fec49758..d245991a 100644 --- a/tests/test_effort.py +++ b/tests/test_effort.py @@ -41,9 +41,9 @@ def test_effort_rejected_for_unsupported_providers(self, provider): with pytest.raises(click.UsageError, match="--effort is not supported"): _provider_kwargs(provider, model="m", effort="low") - def test_effort_rejected_for_synthpanel_with_pointer(self): - with pytest.raises(click.UsageError, match="synthpanel"): - _provider_kwargs("synthpanel", model="haiku", effort="medium") + def test_effort_rejected_for_althing_with_pointer(self): + with pytest.raises(click.UsageError, match="althing"): + _provider_kwargs("althing", model="haiku", effort="medium") def test_absent_effort_adds_no_kwarg(self): kwargs = _provider_kwargs("raw-anthropic", model="m") @@ -261,12 +261,21 @@ class TestConfigIdEffort: "raw--gpt-4o-mini--tdefault--tplcurrent--704154ed", ), ( + # Historical golden: pre-rename provider ids must hash stably + # forever — leaderboard-results/ artifacts reference this id. dict( provider="synthpanel/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current", dataset="opinionsqa", ), "synthpanel--claude-haiku-4-5--t0.85--tplcurrent--9f9ed2bc", ), + ( + dict( + provider="althing/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current", + dataset="opinionsqa", + ), + "althing--claude-haiku-4-5--t0.85--tplcurrent--d22b2768", + ), ( dict( provider="random-baseline", @@ -311,13 +320,11 @@ def test_effort_appears_in_slug_and_knobs(self): assert parsed.knobs["effort"] == "high" def test_effort_knob_parsed_from_provider_string(self): - """A provider string carrying `effort=high` (synthpanel-style knob + """A provider string carrying `effort=high` (althing-style knob tokens) resolves the same as the explicit kwarg.""" - via_knob, _ = build_config_id( - "synthpanel/x/y/z effort=high", dataset="opinionsqa" - ) + via_knob, _ = build_config_id("althing/x/y/z effort=high", dataset="opinionsqa") via_kwarg, _ = build_config_id( - "synthpanel/x/y/z", dataset="opinionsqa", effort="high" + "althing/x/y/z", dataset="opinionsqa", effort="high" ) assert via_knob == via_kwarg diff --git a/tests/test_findings_elicitation.py b/tests/test_findings_elicitation.py index 5a7a768d..5d18cc8f 100644 --- a/tests/test_findings_elicitation.py +++ b/tests/test_findings_elicitation.py @@ -7,7 +7,7 @@ from synthbench.findings import _build_elicitation_comparison, _parse_failure_rate _QHASH = "a" * 40 -_PROVIDER = "synthpanel/openrouter/anthropic/claude-haiku-4-5" +_PROVIDER = "althing/openrouter/anthropic/claude-haiku-4-5" def _q(key: str, jsd: float, tau: float, refusal: float = 0.0) -> dict: @@ -73,7 +73,7 @@ def test_matched_pair_emits_arms_and_deltas(): rows = _build_elicitation_comparison([natural, structured]) assert len(rows) == 1 row = rows[0] - assert row["model"] == "SynthPanel (Haiku 4.5)" + assert row["model"] == "Althing (Haiku 4.5)" assert row["framework"] == "product" assert row["dataset"] == "gss" assert row["elicitation"] == "structured" diff --git a/tests/test_findings_nonresponse.py b/tests/test_findings_nonresponse.py index f9ec5a7a..f2111b32 100644 --- a/tests/test_findings_nonresponse.py +++ b/tests/test_findings_nonresponse.py @@ -74,7 +74,7 @@ def test_build_nonresponse_fidelity_skips_stripped_runs(): def test_sensitive_topic_note_derived_from_worst_raw_row(): rows = [ { - "provider": "SynthPanel (Haiku 4.5)", + "provider": "Althing (Haiku 4.5)", "framework": "product", "dataset": "gss", "n_questions": 75, diff --git a/tests/test_provider_parsing.py b/tests/test_provider_parsing.py index 458ee19a..8e94bec4 100644 --- a/tests/test_provider_parsing.py +++ b/tests/test_provider_parsing.py @@ -257,25 +257,25 @@ def test_constructors_reject_unknown_kwargs(monkeypatch): MajorityBaselineProvider(temperature=0.5) -def test_synthpanel_prompt_template_forces_cli_path(monkeypatch, tmp_path): +def test_althing_prompt_template_forces_cli_path(monkeypatch, tmp_path): """A --prompt-template run must never take the direct API path. The API path builds its own persona/system prompt and has no seam for - the synthpanel template override — it only stamped the template into + the althing template override — it only stamped the template into the provider name and prompt hash, so a ``tpl=`` run would silently execute the default prompt (untrue leaderboard metadata). - Only the CLI path threads --prompt-template through to synthpanel. + Only the CLI path threads --prompt-template through to althing. """ - from synthbench.providers import synthpanel as sp_mod + from synthbench.providers import althing as sp_mod tpl = tmp_path / "custom.txt" tpl.write_text("You are {name}.") - # Even with the synth_panel API importable, a template override must + # Even with the althing API importable, a template override must # route through the CLI. monkeypatch.setattr(sp_mod, "_HAS_SYNTH_PANEL_API", True) - provider = sp_mod.SynthPanelProvider( - model="haiku", prompt_template=str(tpl), synthpanel_path="/bin/echo" + provider = sp_mod.AlthingProvider( + model="haiku", prompt_template=str(tpl), althing_path="/bin/echo" ) assert provider._use_api is False cmd = provider._build_cmd("inst.yaml", "pers.yaml") @@ -283,16 +283,16 @@ def test_synthpanel_prompt_template_forces_cli_path(monkeypatch, tmp_path): assert cmd[idx + 1] == str(tpl) # Template-free runs keep the fast API path. monkeypatch.setattr(sp_mod, "LLMClient", lambda: object(), raising=False) - default_provider = sp_mod.SynthPanelProvider(model="haiku") + default_provider = sp_mod.AlthingProvider(model="haiku") assert default_provider._use_api is True -def test_synthpanel_temperature_reaches_cli_command(monkeypatch, tmp_path): - from synthbench.providers import synthpanel as sp_mod +def test_althing_temperature_reaches_cli_command(monkeypatch, tmp_path): + from synthbench.providers import althing as sp_mod monkeypatch.setattr(sp_mod, "_HAS_SYNTH_PANEL_API", False) - provider = sp_mod.SynthPanelProvider( - model="haiku", temperature=0.4, synthpanel_path="/bin/echo" + provider = sp_mod.AlthingProvider( + model="haiku", temperature=0.4, althing_path="/bin/echo" ) cmd = provider._build_cmd("inst.yaml", "pers.yaml") idx = cmd.index("--temperature") @@ -626,7 +626,7 @@ def test_missing_n_samples_field_not_counted_as_zero(): # --------------------------------------------------------------------------- -# synthpanel CLI fallbacks: infra failures raise, never fabricate +# althing CLI fallbacks: infra failures raise, never fabricate # --------------------------------------------------------------------------- @@ -640,11 +640,11 @@ async def communicate(self): return self._stdout, self._stderr -def _cli_synthpanel_provider(monkeypatch): - from synthbench.providers import synthpanel as sp_mod +def _cli_althing_provider(monkeypatch): + from synthbench.providers import althing as sp_mod monkeypatch.setattr(sp_mod, "_HAS_SYNTH_PANEL_API", False) - return sp_mod.SynthPanelProvider(model="haiku", synthpanel_path="/bin/echo") + return sp_mod.AlthingProvider(model="haiku", althing_path="/bin/echo") def _patch_subprocess(monkeypatch, proc: _FakeProc): @@ -655,26 +655,26 @@ async def fake_exec(*cmd, **kwargs): @pytest.mark.asyncio -async def test_synthpanel_subprocess_failure_raises_not_uniform(monkeypatch): - provider = _cli_synthpanel_provider(monkeypatch) +async def test_althing_subprocess_failure_raises_not_uniform(monkeypatch): + provider = _cli_althing_provider(monkeypatch) _patch_subprocess(monkeypatch, _FakeProc(returncode=1, stderr=b"boom")) with pytest.raises(ProviderError): await provider.get_distribution("Q?", ["Yes", "No"], n_samples=3) @pytest.mark.asyncio -async def test_synthpanel_bad_json_raises_not_uniform(monkeypatch): - provider = _cli_synthpanel_provider(monkeypatch) +async def test_althing_bad_json_raises_not_uniform(monkeypatch): + provider = _cli_althing_provider(monkeypatch) _patch_subprocess(monkeypatch, _FakeProc(returncode=0, stdout=b"not json")) with pytest.raises(ProviderError): await provider.get_distribution("Q?", ["Yes", "No"], n_samples=3) @pytest.mark.asyncio -async def test_synthpanel_batch_classifies_responses(monkeypatch): +async def test_althing_batch_classifies_responses(monkeypatch): import json as _json - provider = _cli_synthpanel_provider(monkeypatch) + provider = _cli_althing_provider(monkeypatch) payload = { "rounds": [ { @@ -701,8 +701,8 @@ async def test_synthpanel_batch_classifies_responses(monkeypatch): @pytest.mark.asyncio -async def test_synthpanel_respond_failure_raises(monkeypatch): - provider = _cli_synthpanel_provider(monkeypatch) +async def test_althing_respond_failure_raises(monkeypatch): + provider = _cli_althing_provider(monkeypatch) _patch_subprocess(monkeypatch, _FakeProc(returncode=2, stderr=b"crashed")) with pytest.raises(ProviderError): await provider.respond("Q?", ["Yes", "No"]) @@ -852,8 +852,8 @@ def test_refusal_detector_version_constant(): def _structured_provider(monkeypatch, answers): - """SynthPanel provider in structured mode with a stubbed API client.""" - from synthbench.providers import synthpanel as sp_mod + """Althing provider in structured mode with a stubbed API client.""" + from synthbench.providers import althing as sp_mod monkeypatch.setattr(sp_mod, "_HAS_SYNTH_PANEL_API", True) @@ -886,7 +886,7 @@ def send(self, request): ) monkeypatch.setattr(sp_mod, "LLMClient", lambda: _FakeClient()) - provider = sp_mod.SynthPanelProvider(model="haiku", elicitation="structured") + provider = sp_mod.AlthingProvider(model="haiku", elicitation="structured") return provider, calls @@ -894,9 +894,9 @@ def test_structured_mode_is_distinct_template_identity(monkeypatch): provider, _ = _structured_provider(monkeypatch, ["Yes"]) assert "tpl=structured" in provider.name # The prompt surface hash must differ from the natural mode's. - from synthbench.providers import synthpanel as sp_mod + from synthbench.providers import althing as sp_mod - natural = sp_mod.SynthPanelProvider.__new__(sp_mod.SynthPanelProvider) + natural = sp_mod.AlthingProvider.__new__(sp_mod.AlthingProvider) natural._model = "haiku" natural._prompt_template = None natural._elicitation = "natural" @@ -904,14 +904,14 @@ def test_structured_mode_is_distinct_template_identity(monkeypatch): def test_structured_mode_rejects_prompt_template(): - from synthbench.providers import synthpanel as sp_mod + from synthbench.providers import althing as sp_mod with pytest.raises(ValueError): - sp_mod.SynthPanelProvider( + sp_mod.AlthingProvider( model="haiku", elicitation="structured", prompt_template="x.md" ) with pytest.raises(ValueError): - sp_mod.SynthPanelProvider(model="haiku", elicitation="nope") + sp_mod.AlthingProvider(model="haiku", elicitation="nope") @pytest.mark.asyncio @@ -946,7 +946,7 @@ async def test_structured_mode_out_of_enum_is_parse_failure(monkeypatch): def test_pick_raw_sample_prefers_modal_option(): - from synthbench.providers.synthpanel import _pick_raw_sample + from synthbench.providers.althing import _pick_raw_sample counts = Counter({"B": 3, "A": 1}) samples = [("I'd say A", "A"), ("Definitely B", "B")] @@ -957,7 +957,7 @@ def test_pick_raw_sample_prefers_modal_option(): def test_pick_raw_sample_falls_back_to_first(): - from synthbench.providers.synthpanel import _pick_raw_sample + from synthbench.providers.althing import _pick_raw_sample counts = Counter({"C": 5}) # No sample matches the modal option — fall back to the first kept one. @@ -1005,7 +1005,7 @@ async def test_batched_runner_propagates_raw_sample(mock_dataset): """The batch path must not drop provider-attached raw samples. Regression: _build_question_result never read metadata['raw_sample'], - so batch-capable distribution providers (synthpanel) shipped schema-v2 + so batch-capable distribution providers (althing) shipped schema-v2 results with empty raw_responses. """ runner = BenchmarkRunner( @@ -1022,11 +1022,11 @@ async def test_batched_runner_propagates_raw_sample(mock_dataset): @pytest.mark.asyncio -async def test_synthpanel_cli_batch_attaches_raw_sample(monkeypatch): - from synthbench.providers import synthpanel as sp_mod +async def test_althing_cli_batch_attaches_raw_sample(monkeypatch): + from synthbench.providers import althing as sp_mod monkeypatch.setattr(sp_mod, "_HAS_SYNTH_PANEL_API", False) - provider = sp_mod.SynthPanelProvider(model="haiku", synthpanel_path="/bin/echo") + provider = sp_mod.AlthingProvider(model="haiku", althing_path="/bin/echo") canned = { "rounds": [ @@ -1054,12 +1054,12 @@ async def _fake_subprocess(cmd): @pytest.mark.asyncio -async def test_synthpanel_api_distribution_attaches_raw_sample(monkeypatch): - from synthbench.providers import synthpanel as sp_mod +async def test_althing_api_distribution_attaches_raw_sample(monkeypatch): + from synthbench.providers import althing as sp_mod - provider = sp_mod.SynthPanelProvider(model="haiku") + provider = sp_mod.AlthingProvider(model="haiku") if not provider._use_api: - pytest.skip("synth_panel API not importable") + pytest.skip("althing API not importable") calls = iter(["(B) No", "(A) Yes", "(A) Yes"]) diff --git a/tests/test_publish_config_id_consistency.py b/tests/test_publish_config_id_consistency.py index 633ddcb2..8d43cd49 100644 --- a/tests/test_publish_config_id_consistency.py +++ b/tests/test_publish_config_id_consistency.py @@ -10,7 +10,7 @@ ``parse_provider``. If either path ever switches to ``provider_framework`` (the human-facing taxonomy: ``raw`` / ``product`` / ``baseline``) for the slug while the other keeps ``parsed.framework`` (the path-derived label: - ``raw`` / ``synthpanel`` / ``ensemble`` / ``baseline``), the prefix + ``raw`` / ``althing`` / ``ensemble`` / ``baseline``), the prefix diverges — e.g. an OpenRouter raw call becomes ``openrouter--gpt-4o-mini`` on one side and ``raw--gpt-4o-mini`` on the other. """ @@ -35,10 +35,10 @@ "openrouter/google/gemini-2.5-flash-lite", "raw-anthropic/claude-haiku-4-5-20251001", "raw-gemini/gemini-2.5-flash-lite", - "synthpanel/openrouter/openai/gpt-4o-mini", - "synthpanel/openrouter/anthropic/claude-haiku-4-5", - "synthpanel/claude-haiku-4-5-20251001", - "synthpanel/gemini-2.5-flash-lite", + "althing/openrouter/openai/gpt-4o-mini", + "althing/openrouter/anthropic/claude-haiku-4-5", + "althing/claude-haiku-4-5-20251001", + "althing/gemini-2.5-flash-lite", "ensemble/3-model-blend", "random-baseline", "majority-baseline", @@ -165,10 +165,10 @@ def test_publish_pipeline_config_ids_match_runs_index(tmp_path): "openrouter/google/gemini-2.5-flash-lite": "raw", "raw-anthropic/claude-haiku-4-5-20251001": "raw", "raw-gemini/gemini-2.5-flash-lite": "raw", - "synthpanel/openrouter/openai/gpt-4o-mini": "synthpanel", - "synthpanel/openrouter/anthropic/claude-haiku-4-5": "synthpanel", - "synthpanel/claude-haiku-4-5-20251001": "synthpanel", - "synthpanel/gemini-2.5-flash-lite": "synthpanel", + "althing/openrouter/openai/gpt-4o-mini": "althing", + "althing/openrouter/anthropic/claude-haiku-4-5": "althing", + "althing/claude-haiku-4-5-20251001": "althing", + "althing/gemini-2.5-flash-lite": "althing", "ensemble/3-model-blend": "ensemble", "random-baseline": "baseline", "majority-baseline": "baseline", diff --git a/tests/test_publish_cost.py b/tests/test_publish_cost.py index cf39d203..3d504154 100644 --- a/tests/test_publish_cost.py +++ b/tests/test_publish_cost.py @@ -18,36 +18,36 @@ ) # Imported lazily by the helpers under test, but the test file needs the -# pricing constants for assertion math. Tests skip cleanly if synthpanel +# pricing constants for assertion math. Tests skip cleanly if althing # (a dep added in this slice) is not installed. -synth_panel_cost = pytest.importorskip("synth_panel.cost") -HAIKU = synth_panel_cost.HAIKU_PRICING -SONNET = synth_panel_cost.SONNET_PRICING +althing_cost = pytest.importorskip("althing.cost") +HAIKU = althing_cost.HAIKU_PRICING +SONNET = althing_cost.SONNET_PRICING # Dynamic read: extract the pricing snapshot_date from the installed -# synth_panel.cost source the same way `_build_pricing_snapshot` does. +# althing.cost source the same way `_build_pricing_snapshot` does. # Previously this test hard-coded "2026-04-14", which broke every time # the upstream cost.py snapshot comment rolled (observed 2026-04-21 when -# synth_panel released v0.9.7 — refinery escalation hq-wisp-ll8ti4). -# Assertions now pin against whatever the installed synth_panel reports, +# althing released v0.9.7 — refinery escalation hq-wisp-ll8ti4). +# Assertions now pin against whatever the installed althing reports, # so the test no longer requires per-release maintenance. _SNAPSHOT_PATTERN = re.compile(r"pricing snapshot_date:\s*(\d{4}-\d{2}-\d{2})") def _current_snapshot_date() -> str: - """Return the snapshot_date reported by the installed synth_panel. + """Return the snapshot_date reported by the installed althing. Reads the same `# pricing snapshot_date: YYYY-MM-DD` anchor comment `_build_pricing_snapshot` parses, from the installed package source. Skips the test (via pytest.skip) if the anchor is missing, which matches the library's own fallback behavior. """ - source = inspect.getsource(synth_panel_cost) + source = inspect.getsource(althing_cost) m = _SNAPSHOT_PATTERN.search(source) if not m: pytest.skip( - "synth_panel.cost does not expose a `pricing snapshot_date:` anchor comment" + "althing.cost does not expose a `pricing snapshot_date:` anchor comment" ) return m.group(1) @@ -252,19 +252,19 @@ def _constituent(provider: str, dataset: str, in_tok: int, out_tok: int) -> dict def test_compute_ensemble_cost_sums_all_constituents(): - haiku = _constituent("synthpanel/claude-haiku-4-5", "subpop", 1_000_000, 500_000) - sonnet = _constituent("synthpanel/claude-sonnet-4", "subpop", 1_000_000, 500_000) + haiku = _constituent("althing/claude-haiku-4-5", "subpop", 1_000_000, 500_000) + sonnet = _constituent("althing/claude-sonnet-4", "subpop", 1_000_000, 500_000) opus = _constituent("raw-anthropic/claude-opus-4-6", "subpop", 1_000_000, 500_000) results_by_pds = { - ("synthpanel/claude-haiku-4-5", "subpop"): haiku, - ("synthpanel/claude-sonnet-4", "subpop"): sonnet, + ("althing/claude-haiku-4-5", "subpop"): haiku, + ("althing/claude-sonnet-4", "subpop"): sonnet, ("raw-anthropic/claude-opus-4-6", "subpop"): opus, } config = { "dataset": "subpop", "ensemble_sources": [ - {"provider": "synthpanel/claude-haiku-4-5", "weight": 1 / 3}, - {"provider": "synthpanel/claude-sonnet-4", "weight": 1 / 3}, + {"provider": "althing/claude-haiku-4-5", "weight": 1 / 3}, + {"provider": "althing/claude-sonnet-4", "weight": 1 / 3}, {"provider": "raw-anthropic/claude-opus-4-6", "weight": 1 / 3}, ], } @@ -276,32 +276,32 @@ def test_compute_ensemble_cost_sums_all_constituents(): (HAIKU.input_cost_per_million + 0.5 * HAIKU.output_cost_per_million) + (SONNET.input_cost_per_million + 0.5 * SONNET.output_cost_per_million) + ( - synth_panel_cost.OPUS_PRICING.input_cost_per_million - + 0.5 * synth_panel_cost.OPUS_PRICING.output_cost_per_million + althing_cost.OPUS_PRICING.input_cost_per_million + + 0.5 * althing_cost.OPUS_PRICING.output_cost_per_million ) ) assert cost == pytest.approx(expected, rel=1e-5) def test_compute_ensemble_cost_missing_constituent_returns_none(): - haiku = _constituent("synthpanel/claude-haiku-4-5", "subpop", 1_000_000, 500_000) + haiku = _constituent("althing/claude-haiku-4-5", "subpop", 1_000_000, 500_000) # opus exists, but sonnet's token_usage is missing AND its config lacks # samples_per_question/n_evaluated, so the #316 estimate can't run either. sonnet_no_usage = { - "config": {"provider": "synthpanel/claude-sonnet-4", "dataset": "subpop"}, + "config": {"provider": "althing/claude-sonnet-4", "dataset": "subpop"}, "aggregate": {"n_questions": 100}, # no token_usage } opus = _constituent("raw-anthropic/claude-opus-4-6", "subpop", 1_000_000, 500_000) results_by_pds = { - ("synthpanel/claude-haiku-4-5", "subpop"): haiku, - ("synthpanel/claude-sonnet-4", "subpop"): sonnet_no_usage, + ("althing/claude-haiku-4-5", "subpop"): haiku, + ("althing/claude-sonnet-4", "subpop"): sonnet_no_usage, ("raw-anthropic/claude-opus-4-6", "subpop"): opus, } config = { "dataset": "subpop", "ensemble_sources": [ - {"provider": "synthpanel/claude-haiku-4-5", "weight": 1 / 3}, - {"provider": "synthpanel/claude-sonnet-4", "weight": 1 / 3}, + {"provider": "althing/claude-haiku-4-5", "weight": 1 / 3}, + {"provider": "althing/claude-sonnet-4", "weight": 1 / 3}, {"provider": "raw-anthropic/claude-opus-4-6", "weight": 1 / 3}, ], } @@ -315,10 +315,10 @@ def test_compute_ensemble_cost_no_sources_returns_none(): def test_compute_ensemble_cost_estimates_unmeasured_constituent(): """#316: a constituent without token_usage contributes the documented samples×questions×per-call-constant estimate and flips is_estimated.""" - haiku = _constituent("synthpanel/claude-haiku-4-5", "subpop", 1_000_000, 500_000) + haiku = _constituent("althing/claude-haiku-4-5", "subpop", 1_000_000, 500_000) sonnet_estimable = { "config": { - "provider": "synthpanel/claude-sonnet-4", + "provider": "althing/claude-sonnet-4", "dataset": "subpop", "samples_per_question": 5, "n_evaluated": 100, @@ -326,14 +326,14 @@ def test_compute_ensemble_cost_estimates_unmeasured_constituent(): "aggregate": {"n_questions": 100}, # no token_usage } results_by_pds = { - ("synthpanel/claude-haiku-4-5", "subpop"): haiku, - ("synthpanel/claude-sonnet-4", "subpop"): sonnet_estimable, + ("althing/claude-haiku-4-5", "subpop"): haiku, + ("althing/claude-sonnet-4", "subpop"): sonnet_estimable, } config = { "dataset": "subpop", "ensemble_sources": [ - {"provider": "synthpanel/claude-haiku-4-5", "weight": 0.5}, - {"provider": "synthpanel/claude-sonnet-4", "weight": 0.5}, + {"provider": "althing/claude-haiku-4-5", "weight": 0.5}, + {"provider": "althing/claude-sonnet-4", "weight": 0.5}, ], } result = _compute_ensemble_cost(config, results_by_pds) @@ -357,7 +357,7 @@ def test_build_pricing_snapshot_shape_and_rates(): snap = _build_pricing_snapshot() assert set(snap.keys()) == { "generated_at", - "synth_panel_version", + "althing_version", "snapshot_date", "rates", } @@ -375,7 +375,7 @@ def test_build_pricing_snapshot_shape_and_rates(): assert rates["haiku"]["output_cost_per_million"] == HAIKU.output_cost_per_million assert rates["sonnet"]["input_cost_per_million"] == SONNET.input_cost_per_million # snapshot_date is read from the installed cost.py anchor comment; - # pin against whatever synth_panel reports today so future releases + # pin against whatever althing reports today so future releases # don't break this test (see module-level _current_snapshot_date). assert snap["snapshot_date"] == CURRENT_SNAPSHOT_DATE assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", snap["snapshot_date"]), ( @@ -471,7 +471,7 @@ def test_golden_three_entry_leaderboard(tmp_path: Path): """Feed 3 raw artifacts (measured / no-tokens / ensemble) through publish_leaderboard_data and assert the cost-related shape and values.""" haiku_with_tokens = _make_run( - provider="synthpanel/claude-haiku-4-5", + provider="althing/claude-haiku-4-5", dataset="subpop", sps=0.83, n_questions=200, @@ -484,7 +484,7 @@ def test_golden_three_entry_leaderboard(tmp_path: Path): }, ) gemini_no_tokens = _make_run( - provider="synthpanel/gemini-2.5-flash-lite", + provider="althing/gemini-2.5-flash-lite", dataset="subpop", sps=0.78, n_questions=200, @@ -500,12 +500,12 @@ def test_golden_three_entry_leaderboard(tmp_path: Path): ensemble_sources=[ { "file": "h.json", - "provider": "synthpanel/claude-haiku-4-5", + "provider": "althing/claude-haiku-4-5", "weight": 0.5, }, { "file": "g.json", - "provider": "synthpanel/gemini-2.5-flash-lite", + "provider": "althing/gemini-2.5-flash-lite", "weight": 0.5, }, ], @@ -545,7 +545,7 @@ def test_golden_three_entry_leaderboard(tmp_path: Path): # Gemini row (no token_usage) — #316 estimate from config # (samples_per_question=5 × n_evaluated=200 = 1000 calls at the # documented per-call token constants), clearly marked estimated. - FLASH_LITE = synth_panel_cost.GEMINI_FLASH_LITE_PRICING + FLASH_LITE = althing_cost.GEMINI_FLASH_LITE_PRICING expected_gemini_cost = ( 1000 * ( @@ -660,7 +660,7 @@ def test_compute_latency_fields_absent_block_returns_nulls(): def test_publish_surfaces_latency_and_per_response_columns(tmp_path: Path): """End-to-end: a run with token_usage + latency block → entry has all sb-293 fields.""" instrumented = _make_run( - provider="synthpanel/claude-haiku-4-5", + provider="althing/claude-haiku-4-5", dataset="subpop", sps=0.83, n_questions=200, diff --git a/tests/test_publish_cross_provider_jsd.py b/tests/test_publish_cross_provider_jsd.py index bfe1dd39..ea2ff13e 100644 --- a/tests/test_publish_cross_provider_jsd.py +++ b/tests/test_publish_cross_provider_jsd.py @@ -173,7 +173,7 @@ def test_non_raw_frameworks_excluded(): per_question=[_pq("q1", {"A": 0.5, "B": 0.5})], ), _result( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", "opinionsqa", per_question=[_pq("q1", {"A": 0.9, "B": 0.1})], ), @@ -182,7 +182,7 @@ def test_non_raw_frameworks_excluded(): models = out["opinionsqa"]["models"] # Only the two raw LLMs survive the framework filter. assert len(models) == 2 - assert all("SynthPanel" not in m for m in models) + assert all("Althing" not in m for m in models) assert all("baseline" not in m.lower() for m in models) diff --git a/tests/test_publish_demographic_scorecard.py b/tests/test_publish_demographic_scorecard.py index d405dfa0..9145c570 100644 --- a/tests/test_publish_demographic_scorecard.py +++ b/tests/test_publish_demographic_scorecard.py @@ -32,7 +32,7 @@ def _run(demographic_breakdown: dict | None) -> dict: "benchmark": "synthbench", "config": { "dataset": "subpop", - "provider": "synthpanel/openrouter/anthropic/claude-haiku-4-5", + "provider": "althing/openrouter/anthropic/claude-haiku-4-5", "n_evaluated": 2, }, "scores": {}, diff --git a/tests/test_publish_invalid_runs.py b/tests/test_publish_invalid_runs.py index 4df6c08f..06f097c7 100644 --- a/tests/test_publish_invalid_runs.py +++ b/tests/test_publish_invalid_runs.py @@ -100,7 +100,7 @@ def test_publish_leaderboard_data_excludes_invalid_runs(tmp_path): sps=0.77, ) bad = _mk_result( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", "subpop", [_uniform_pq(f"Q{i}") for i in range(20)], sps=0.677, @@ -118,13 +118,13 @@ def test_publish_leaderboard_data_excludes_invalid_runs(tmp_path): assert len(payload["excluded_runs"]) == 1 excluded = payload["excluded_runs"][0] assert excluded["run_id"] == "bad_uniform_garbage" - assert excluded["provider"] == "synthpanel/claude-haiku-4-5-20251001" + assert excluded["provider"] == "althing/claude-haiku-4-5-20251001" assert excluded["metrics"]["n_uniform_questions"] == 20 assert "uniform-garbage" in excluded["reason"] # The invalid run does NOT appear in ranked entries. entry_providers = [e["provider"] for e in payload["entries"]] - assert not any("synthpanel" in p.lower() for p in entry_providers) + assert not any("althing" in p.lower() for p in entry_providers) assert len(payload["entries"]) == 1 @@ -151,7 +151,7 @@ def test_publish_leaderboard_data_all_invalid_raises(tmp_path): results_dir = tmp_path / "results" results_dir.mkdir() bad = _mk_result( - "synthpanel/broken", + "althing/broken", "subpop", [_uniform_pq(f"Q{i}") for i in range(20)], ) @@ -180,7 +180,7 @@ def test_publish_runs_skips_invalid_runs(tmp_path): [_healthy_pq(f"Q{i}") for i in range(20)], ) bad = _mk_result( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", "subpop", [_uniform_pq(f"Q{i}") for i in range(20)], ) @@ -215,7 +215,7 @@ def test_publish_questions_skips_invalid_runs(tmp_path): [_healthy_pq("Q1"), _healthy_pq("Q2")], ) bad = _mk_result( - "synthpanel/claude-haiku-4-5-20251001", + "althing/claude-haiku-4-5-20251001", "ntia", [_uniform_pq(f"Q{i}") for i in range(20)], ) diff --git a/tests/test_publish_normalized.py b/tests/test_publish_normalized.py index 72be5990..501ce4a1 100644 --- a/tests/test_publish_normalized.py +++ b/tests/test_publish_normalized.py @@ -36,7 +36,7 @@ def test_product_normalization_uses_matching_raw_base(): "sps": 0.815, }, { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "dataset": "opinionsqa", "framework": "product", "sps": 0.829, @@ -50,7 +50,7 @@ def test_product_normalization_uses_matching_raw_base(): def test_product_without_known_base_is_skipped(): entries = [ { - "model": "SynthPanel Ensemble (3-model)", + "model": "Althing Ensemble (3-model)", "dataset": "opinionsqa", "framework": "product", "sps": 0.835, @@ -88,7 +88,7 @@ def test_missing_ceiling_skips_dataset(): "sps": 0.72, }, { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "dataset": "globalopinionqa", "framework": "product", "sps": 0.73, @@ -110,7 +110,7 @@ def test_product_without_matching_raw_on_same_dataset_is_skipped(): "sps": 0.815, }, { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "dataset": "subpop", "framework": "product", "sps": 0.77, @@ -130,7 +130,7 @@ def test_clamps_to_range_when_sps_exceeds_ceiling(): "sps": 0.815, }, { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "dataset": "opinionsqa", "framework": "product", "sps": 1.2, @@ -151,7 +151,7 @@ def test_non_positive_range_skipped(): "sps": 0.9995, }, { - "model": "SynthPanel (Haiku 4.5)", + "model": "Althing (Haiku 4.5)", "dataset": "opinionsqa", "framework": "product", "sps": 0.999, diff --git a/tests/test_publish_questions.py b/tests/test_publish_questions.py index 9f65f20c..85597103 100644 --- a/tests/test_publish_questions.py +++ b/tests/test_publish_questions.py @@ -116,7 +116,7 @@ def test_collect_rollups_partitions_ensembles_and_skips_baselines(): # Aggregated bucket: ensemble lands here, not in model_responses. aggregated = rollup["aggregated_responses"] assert len(aggregated) == 1 - assert aggregated[0]["model"] == "SynthPanel Ensemble (3-model)" + assert aggregated[0]["model"] == "Althing Ensemble (3-model)" assert aggregated[0]["run_id"] == "ensemble_run" # Ensembles are product-framework in MODEL_MAP; confirm they're not # leaking into the raw/product single-model bucket. @@ -420,7 +420,7 @@ def test_finalize_emits_aggregated_responses_excluded_from_summary(): "aggregated_responses": [ { "config_id": "cfg-ens", - "model": "SynthPanel Ensemble (3-model)", + "model": "Althing Ensemble (3-model)", "framework": "product", "base_provider": "3-model-blend", "distribution": {"A": 0.5, "B": 0.5}, @@ -652,7 +652,7 @@ def test_publish_questions_writes_expected_layout(tmp_path): ] # Ensemble lives in the aggregated bucket, not model_responses. assert [r["model"] for r in q1["aggregated_responses"]] == [ - "SynthPanel Ensemble (3-model)", + "Althing Ensemble (3-model)", ] assert all( "ensemble" not in r["base_provider"].lower() for r in q1["model_responses"] diff --git a/tests/test_publish_run_counts.py b/tests/test_publish_run_counts.py index a50b3674..054f6815 100644 --- a/tests/test_publish_run_counts.py +++ b/tests/test_publish_run_counts.py @@ -131,13 +131,13 @@ def test_suffixed_providers_collapse_to_display_name(): # display_provider_name and count against the base model row. results = [ _result( - "synthpanel/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current", + "althing/openrouter/anthropic/claude-haiku-4-5 t=0.85 tpl=current", "opinionsqa", temperature=0.85, prompt_template="current", ), _result( - "synthpanel/openrouter/anthropic/claude-haiku-4-5", + "althing/openrouter/anthropic/claude-haiku-4-5", "opinionsqa", temperature=0.85, prompt_template="current", @@ -145,7 +145,7 @@ def test_suffixed_providers_collapse_to_display_name(): ] entries = [ _entry( - "SynthPanel (Haiku 4.5)", + "Althing (Haiku 4.5)", "product", "opinionsqa", temperature=0.85, diff --git a/tests/test_publish_runnable_ids.py b/tests/test_publish_runnable_ids.py index 97951749..4df93871 100644 --- a/tests/test_publish_runnable_ids.py +++ b/tests/test_publish_runnable_ids.py @@ -1,12 +1,12 @@ """Runnable provider/model IDs in leaderboard.json entries (sb-7ly / #297). ``/data/leaderboard.json`` historically put display labels (e.g. -``SynthPanel (Gemini Flash Lite)``) in both ``provider`` and ``model``. -Consumers (SynthPanel #519) need *runnable* IDs so they can pipe a row's model +``Althing (Gemini Flash Lite)``) in both ``provider`` and ``model``. +Consumers (Althing #519) need *runnable* IDs so they can pipe a row's model straight to OpenRouter without parsing display names. ``_build_entry`` now also emits: -- ``provider_id`` — framework/runner (raw / synthpanel / ensemble / baseline) +- ``provider_id`` — framework/runner (raw / althing / ensemble / baseline) - ``model_id`` — an OpenRouter ``/`` slug This locks the exact ID per production provider shape. The key correctness @@ -45,36 +45,36 @@ "anthropic/claude-haiku-4-5-20251001", ), ("raw-gemini/gemini-2.5-flash-lite", "raw", "google/gemini-2.5-flash-lite"), - # SynthPanel product layer over an OpenRouter-routed model. + # Althing product layer over an OpenRouter-routed model. ( - "synthpanel/openrouter/openai/gpt-4o-mini", - "synthpanel", + "althing/openrouter/openai/gpt-4o-mini", + "althing", "openai/gpt-4o-mini", ), ( - "synthpanel/openrouter/anthropic/claude-haiku-4-5", - "synthpanel", + "althing/openrouter/anthropic/claude-haiku-4-5", + "althing", "anthropic/claude-haiku-4-5", ), ( - "synthpanel/openrouter/anthropic/claude-sonnet-4", - "synthpanel", + "althing/openrouter/anthropic/claude-sonnet-4", + "althing", "anthropic/claude-sonnet-4", ), ( - "synthpanel/openrouter/google/gemini-2.5-flash-lite", - "synthpanel", + "althing/openrouter/google/gemini-2.5-flash-lite", + "althing", "google/gemini-2.5-flash-lite", ), - # SynthPanel over a direct (bare) model — vendor inferred so the slug is + # Althing over a direct (bare) model — vendor inferred so the slug is # still OpenRouter-pipeable. ( - "synthpanel/claude-haiku-4-5-20251001", - "synthpanel", + "althing/claude-haiku-4-5-20251001", + "althing", "anthropic/claude-haiku-4-5-20251001", ), - ("synthpanel/gemini-2.5-flash-lite", "synthpanel", "google/gemini-2.5-flash-lite"), - ("synthpanel/gpt-4o-mini", "synthpanel", "openai/gpt-4o-mini"), + ("althing/gemini-2.5-flash-lite", "althing", "google/gemini-2.5-flash-lite"), + ("althing/gpt-4o-mini", "althing", "openai/gpt-4o-mini"), # Ensemble + baselines have no OpenRouter equivalent — keep the native id. ("ensemble/3-model-blend", "ensemble", "3-model-blend"), ("random-baseline", "baseline", "random-baseline"), @@ -120,8 +120,8 @@ def test_runnable_ids_per_provider_shape(): def test_runnable_ids_ignore_hyperparameter_knobs(): """t=/tpl=/profile= suffixes don't change the runnable IDs.""" - base = "synthpanel/openrouter/anthropic/claude-haiku-4-5" - expected = ("synthpanel", "anthropic/claude-haiku-4-5") + base = "althing/openrouter/anthropic/claude-haiku-4-5" + expected = ("althing", "anthropic/claude-haiku-4-5") assert runnable_ids(base) == expected assert runnable_ids(f"{base} t=0.85") == expected assert runnable_ids(f"{base} t=0.85 tpl=current") == expected diff --git a/tests/test_run_validity.py b/tests/test_run_validity.py index bcd19614..c9bb8b87 100644 --- a/tests/test_run_validity.py +++ b/tests/test_run_validity.py @@ -19,7 +19,7 @@ LEADERBOARD_DIR = Path(__file__).resolve().parent.parent / "leaderboard-results" BAD_RUN = ( - LEADERBOARD_DIR / "subpop_synthpanel_claude-haiku-4-5-20251001_20260411_073013.json" + LEADERBOARD_DIR / "subpop_althing_claude-haiku-4-5-20251001_20260411_073013.json" ) GOOD_RUN_GLOB = "globalopinionqa_openrouter_anthropic_claude-haiku-4-5_*.json" diff --git a/tests/test_stats_golden.py b/tests/test_stats_golden.py index ec1b6308..ea7d6a0b 100644 --- a/tests/test_stats_golden.py +++ b/tests/test_stats_golden.py @@ -1,4 +1,4 @@ -"""Golden tests for vendored stats.py — values must match synthpanel exactly.""" +"""Golden tests for vendored stats.py — values must match althing exactly.""" from __future__ import annotations @@ -15,7 +15,7 @@ class TestKrippendorffGolden: """4-rater 12-item nominal example with missing data. Verified against krippendorff 0.8.1 reference package: alpha = 0.871. - Data matches synthpanel's test_stats.py exactly. + Data matches althing's test_stats.py exactly. """ RELIABILITY_DATA = [ diff --git a/tests/test_user_config.py b/tests/test_user_config.py index e12e5bb1..ac7cc717 100644 --- a/tests/test_user_config.py +++ b/tests/test_user_config.py @@ -31,7 +31,7 @@ def _valid_dict() -> dict: return { - "vendor": "synthpanel", + "vendor": "althing", "vendor_version": "1.4.0", "config_name": "wesley-healthcare-nc", "contributor": "@wesley", @@ -82,7 +82,7 @@ def test_load_non_mapping(tmp_path): def test_load_invalid_yaml(tmp_path): p = tmp_path / "bad.yaml" - p.write_text("vendor: synthpanel\n bad: indent\n") + p.write_text("vendor: althing\n bad: indent\n") res = uc.load_config_file(p) assert isinstance(res, uc.ConfigError) assert "invalid YAML" in res.message @@ -96,7 +96,7 @@ def test_load_invalid_yaml(tmp_path): def test_valid_config_returns_user_config(): result = uc.validate_config(_valid_dict()) assert isinstance(result, uc.UserConfig) - assert result.leaderboard_row_key == "synthpanel/wesley-healthcare-nc" + assert result.leaderboard_row_key == "althing/wesley-healthcare-nc" assert result.contributor == "@wesley" @@ -249,12 +249,12 @@ def test_validation_collects_all_errors(): def test_attach_to_submission_body_adds_user_config(): cfg = uc.validate_config(_valid_dict()) assert isinstance(cfg, uc.UserConfig) - body = json.dumps({"benchmark": "synthbench", "config": {"provider": "synthpanel"}}) + body = json.dumps({"benchmark": "synthbench", "config": {"provider": "althing"}}) stamped = uc.attach_to_submission_body(body, cfg) parsed = json.loads(stamped) assert parsed["benchmark"] == "synthbench" - assert parsed["config"] == {"provider": "synthpanel"} - assert parsed["user_config"]["vendor"] == "synthpanel" + assert parsed["config"] == {"provider": "althing"} + assert parsed["user_config"]["vendor"] == "althing" assert parsed["user_config"]["config_name"] == "wesley-healthcare-nc" @@ -355,7 +355,7 @@ def test_cli_validate_config_success(tmp_path): res = CliRunner().invoke(main, ["validate-config", str(p)]) assert res.exit_code == 0, res.output assert "valid" in res.output - assert "synthpanel/wesley-healthcare-nc" in res.output + assert "althing/wesley-healthcare-nc" in res.output def test_cli_validate_config_failure(tmp_path): @@ -388,7 +388,7 @@ def _write_run(tmp_path): json.dumps( { "benchmark": "synthbench", - "config": {"provider": "synthpanel"}, + "config": {"provider": "althing"}, "aggregate": {"n_questions": 1, "composite_parity": 0.7}, "per_question": [ { @@ -429,11 +429,11 @@ def fake_post(url, content=None, headers=None, timeout=None): assert res.exit_code == 0, res.output assert "attaching user_config" in res.output sent = json.loads(captured["content"]) - assert sent["user_config"]["vendor"] == "synthpanel" + assert sent["user_config"]["vendor"] == "althing" assert sent["user_config"]["config_name"] == "wesley-healthcare-nc" # The existing body must be preserved. assert sent["benchmark"] == "synthbench" - assert sent["config"] == {"provider": "synthpanel"} + assert sent["config"] == {"provider": "althing"} def test_cli_submit_config_validation_failure_exits_2(tmp_path, monkeypatch): diff --git a/workers/data-proxy/src/submit.ts b/workers/data-proxy/src/submit.ts index a6ed4f27..3e771144 100644 --- a/workers/data-proxy/src/submit.ts +++ b/workers/data-proxy/src/submit.ts @@ -96,7 +96,7 @@ export function validateTier1(raw: unknown): ValidateResult { // markup/control characters at ingest so a crafted value never reaches the // published catalog in the first place. The allowlist is deliberately // generous — Unicode letters + digits + common punctuation — so real model - // names like "google/gemini-2.5-flash-lite" or "SynthPanel (3-model)" pass. + // names like "google/gemini-2.5-flash-lite" or "Althing (3-model)" pass. for (const field of ["model", "provider", "framework", "dataset"] as const) { const value = config[field]; if (typeof value === "string" && !SAFE_META_RE.test(value)) { diff --git a/workers/data-proxy/tests/submit.test.ts b/workers/data-proxy/tests/submit.test.ts index 9ef06dea..dc6ba82f 100644 --- a/workers/data-proxy/tests/submit.test.ts +++ b/workers/data-proxy/tests/submit.test.ts @@ -162,7 +162,7 @@ describe("validateTier1", () => { baseSubmission({ config: { model: "google/gemini-2.5-flash-lite", - provider: "SynthPanel Ensemble (3-model)", + provider: "Althing Ensemble (3-model)", dataset: "globalopinionqa", framework: "meta-llama/llama-3.3-70b-instruct", },