From 40fb551460bd02c2718878ab4f7c4b2b69c558df Mon Sep 17 00:00:00 2001 From: Rodolphe Lefebvre Date: Thu, 25 Jun 2026 14:22:31 +0200 Subject: [PATCH] Redesign Templates as a catalog-style browser Recompose /templates around a browse-first catalog shell with category buckets, grouped cards, preview state, and embedded create/modify authoring. Keep benchmark template CRUD, AI draft propagation, Advanced form, Raw JSON, and e2e coverage aligned with the new flow. --- CHANGELOG.md | 1 + README.md | 2 +- frontend/src/pages/Templates.tsx | 434 ++++++++++++++------- frontend/src/styles/index.css | 435 +++++++++++++++++++++- frontend/tests/e2e/templates-crud.spec.ts | 34 +- 5 files changed, 747 insertions(+), 159 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7970003..036a002 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ The format is based on Keep a Changelog and this project follows Semantic Versio - **Human-readable agent workflow guidance** — `AGENTS.md` now groups workflow rules into clearer sections, documents parallel worktree expectations including `origin/main` checks before commit/push requests and resync timing before validation or merge, directs agents to create a focused branch without pausing for confirmation, and asks agents to explicitly request commit approval with a suggested message and details. - **Templates agent composer** — The Templates authoring panel now gives the freeform request field more space and removes the preset suggestion chip. - **Run selection rail polish** — the Run page model chips, benchmark template selector, and response header now use clearer selected-state borders/backgrounds and denser mono text, with redundant server summary and model helper copy removed. +- **Templates catalog shell** — `/templates` now opens as a browse-first catalog with category buckets, grouped template cards, dedicated preview state, and the existing AI-first authoring flow embedded in the new shell instead of auto-selecting the first template row. ### Fixed diff --git a/README.md b/README.md index 5d3ebc3..ab56653 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Start with built-in benchmark templates, then create tests for one prompt, a dat Benchmark documents are persisted as JSON in a file-backed library and indexed into SQLite for runtime use. Built-in documents ship with the app, while user-created templates, datasets, runtime profiles, and plans are written to a local library directory so they can be restored if the database is rebuilt. **Benchmark template agent** -Use the Templates page agent as the primary authoring flow to challenge underspecified benchmark ideas, draft runnable `chat_completion` benchmark templates, validate them against the benchmark schema, and review the live JSON before saving. The full structured editor remains available as an Advanced escape hatch. +Use the Templates catalog page agent as the primary authoring flow to challenge underspecified benchmark ideas, draft runnable `chat_completion` benchmark templates, validate them against the benchmark schema, browse the library by category, and review the live JSON before saving. The full structured editor remains available as an Advanced escape hatch. **Benchmark runs** Run the same test against one model, many models, or the same model served by different inference servers. diff --git a/frontend/src/pages/Templates.tsx b/frontend/src/pages/Templates.tsx index d163078..bab4e6f 100644 --- a/frontend/src/pages/Templates.tsx +++ b/frontend/src/pages/Templates.tsx @@ -17,8 +17,9 @@ import { TOOL_CALL_ASSERTION_METRIC } from '../services/benchmark-metric-metadat import { listModels, type ModelRecord } from '../services/models-api.js'; import { getAppSettings, type AppSettings } from '../services/system-api.js'; -type TemplateMode = { kind: 'preview' } | { kind: 'create' } | { kind: 'modify' }; +type TemplateMode = { kind: 'grid' } | { kind: 'preview' } | { kind: 'create' } | { kind: 'modify' }; type TemplateOperationFilter = 'all' | BenchmarkOperation; +type TemplateCategory = 'all' | 'tool' | 'agentic' | 'structured'; type AuthorTab = 'live' | 'advanced' | 'raw'; type DraftSource = 'none' | 'seed' | 'agent' | 'advanced' | 'raw'; @@ -34,8 +35,20 @@ interface ChatEntry { questions?: string[]; } +interface TemplateCategoryDefinition { + id: TemplateCategory; + name: string; + blurb: string; +} + const OPERATION_FILTERS: TemplateOperationFilter[] = ['all', 'chat_completion', 'completion', 'embedding', 'list_models', 'healthcheck']; const ENABLED_OPERATION_FILTERS = new Set(['all', 'chat_completion']); +const TEMPLATE_CATEGORIES: TemplateCategoryDefinition[] = [ + { id: 'all', name: 'All templates', blurb: 'Every benchmark in the workspace' }, + { id: 'tool', name: 'Tool calling', blurb: 'Selection, arguments, and tool-choice mechanics' }, + { id: 'agentic', name: 'Agentic', blurb: 'Multi-step agents, repo edits, and workflow execution' }, + { id: 'structured', name: 'Structured output', blurb: 'JSON contracts and schema validation' } +]; const DEFAULT_TEMPLATE_DRAFT: BenchmarkTestTemplateDocument = { kind: 'test_template', @@ -92,6 +105,27 @@ function parseTemplateStats(template: BenchmarkTestTemplateRecord): { }; } +function categorizeTemplateDocument(document: BenchmarkTestTemplateDocument): Exclude { + const id = document.template_id.toLowerCase(); + if (id.startsWith('agent-')) { + return 'agentic'; + } + if (id.startsWith('functional-')) { + return 'structured'; + } + if (document.required_capabilities?.structured_output && !document.required_capabilities?.tool_calling) { + return 'structured'; + } + if (document.required_capabilities?.tool_calling) { + return 'tool'; + } + return 'tool'; +} + +function categoryName(category: TemplateCategory): string { + return TEMPLATE_CATEGORIES.find((entry) => entry.id === category)?.name ?? 'Templates'; +} + function formatDate(value: string): string { const date = new Date(value); return Number.isNaN(date.getTime()) ? value : date.toLocaleDateString(); @@ -190,71 +224,146 @@ function JsonView({ lines, showNotes }: { lines: JsonLine[]; showNotes: boolean ); } -function TemplatesRail({ +function TemplateCategoryRail({ + category, + counts, + onCategory +}: { + category: TemplateCategory; + counts: Record; + onCategory: (value: TemplateCategory) => void; +}) { + return ( + + ); +} + +function TemplateCard({ + template, + onSelect +}: { + template: BenchmarkTestTemplateRecord; + onSelect: (templateId: string) => void; +}) { + return ( + + ); +} + +function TemplateBrowseGrid({ templates, - selectedId, + category, query, operationFilter, - mode, - busy, onQuery, onFilter, onSelect, onNew }: { templates: BenchmarkTestTemplateRecord[]; - selectedId: string | null; + category: TemplateCategory; query: string; operationFilter: TemplateOperationFilter; - mode: TemplateMode; - busy: boolean; onQuery: (value: string) => void; onFilter: (value: TemplateOperationFilter) => void; - onSelect: (id: string) => void; + onSelect: (templateId: string) => void; onNew: () => void; }) { + const groupedSections = category === 'all' + ? TEMPLATE_CATEGORIES + .filter((entry): entry is TemplateCategoryDefinition & { id: Exclude } => entry.id !== 'all') + .map((entry) => ({ + category: entry, + items: templates.filter((template) => categorizeTemplateDocument(template.document) === entry.id) + })) + .filter((section) => section.items.length > 0) + : []; + return ( - + +
+ {templates.length === 0 ? ( +
+

No templates match

+

Try a different category, clear the search, or create a new template.

+ +
+ ) : category === 'all' ? ( + groupedSections.map((section) => ( +
+
+ {section.category.name} + {section.items.length} + +
+
+ {section.items.map((template) => )} +
+
+ )) + ) : ( +
+ {templates.map((template) => )} +
+ )} +
+ ); } @@ -294,10 +403,12 @@ function TemplateJsonWindow({ function TemplatePreview({ template, + onBack, onModify, onDelete }: { template: BenchmarkTestTemplateRecord; + onBack: () => void; onModify: () => void; onDelete: () => void; }) { @@ -307,6 +418,7 @@ function TemplatePreview({ const includesToolCallAssertion = document.metrics.includes(TOOL_CALL_ASSERTION_METRIC); return (
+
benchmark @@ -490,6 +602,13 @@ function TemplateAgentPanel({ ) : null}
+ {currentDraft ? ( +
+ Validated draft / passes benchmark_test_template_v1 +

{currentDraft.name ?? currentDraft.template_id} / {currentDraft.operation} / {currentDraft.metrics.length} metrics

+
+ ) : null} +