diff --git a/evals/deck-review-report.test.mjs b/evals/deck-review-report.test.mjs new file mode 100644 index 0000000..7218428 --- /dev/null +++ b/evals/deck-review-report.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { buildReport } from '../plugins/visual-explainer/scripts/verify/lib/report.mjs'; + +function context(overrides = {}) { + return { + filePath: '/tmp/artifact.html', + profile: 'page', + preset: 'custom', + presetHint: 'custom', + html: '
ordinary page
', + ...overrides, + }; +} + +test('routes fixed-stage presentations through the dedicated deck review pass', () => { + const report = buildReport(context({ + html: '
', + }), []); + assert.ok(report.llm_passes_required.includes('deck-review')); + assert.equal( + report.llm_dispatch_plan.find((entry) => entry.pass === 'deck-review')?.reason, + 'no-eval-qualified-model', + ); +}); + +test('routes slide decks but leaves magazines and ordinary pages alone', () => { + const slides = buildReport(context({ profile: 'slides' }), []); + assert.ok(slides.llm_passes_required.includes('deck-review')); + + const magazine = buildReport(context({ profile: 'magazine' }), []); + assert.equal(magazine.llm_passes_required.includes('deck-review'), false); + + const page = buildReport(context(), []); + assert.equal(page.llm_passes_required.includes('deck-review'), false); +}); diff --git a/evals/deck-review-set.browser.test.mjs b/evals/deck-review-set.browser.test.mjs new file mode 100644 index 0000000..0a95532 --- /dev/null +++ b/evals/deck-review-set.browser.test.mjs @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { runBrowserStage } from '../plugins/visual-explainer/scripts/verify/lib/browser.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const FIXTURE = path.join(ROOT, 'evals/fixtures/deck-review/presentation-states.html'); + +test('browser stage captures every presentation base, drill, and progressive state', async () => { + const screensDir = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-deck-review-')); + const html = fs.readFileSync(FIXTURE, 'utf8'); + const ctx = { + filePath: FIXTURE, + html, + profile: 'page', + preset: 'custom', + flags: { hasAnimations: false, hasMermaid: false }, + }; + + try { + const browser = await runBrowserStage(ctx, { screensDir, profile: 'page' }); + assert.equal(browser.runs.length, 4, 'the frozen page browser matrix remains unchanged'); + const reviewSets = fs.readdirSync(screensDir) + .filter((name) => /^deck-review-.*\.json$/.test(name)) + .sort() + .map((name) => JSON.parse(fs.readFileSync(path.join(screensDir, name), 'utf8'))); + assert.equal(reviewSets.length, 2, 'light and dark desktop runs receive sidecar review sets'); + for (const reviewSet of reviewSets) { + assert.deepEqual( + reviewSet.units.map((unit) => unit.state_id), + [ + 'slide-one--base', + 'slide-one--drill--evidence', + 'slide-two--base', + 'slide-two--state--1', + 'slide-two--drill--state-1-progressive-evidence', + ], + ); + assert.ok(reviewSet.units.every((unit) => fs.existsSync(unit.screenshot_path))); + assert.deepEqual(reviewSet.viewport, { width: 1440, height: 900 }); + assert.deepEqual( + reviewSet.review_groups.map((group) => [group.purpose, group.state_ids]), + [ + ['state-continuity', ['slide-one--base', 'slide-one--drill--evidence']], + ['state-continuity', ['slide-two--base', 'slide-two--state--1']], + ['state-continuity', ['slide-two--base', 'slide-two--drill--state-1-progressive-evidence']], + ['adjacent-slide-variety', ['slide-one--base', 'slide-two--base']], + ], + ); + } + } finally { + fs.rmSync(screensDir, { recursive: true, force: true }); + } +}); + +test('bounded recapture fails when any requested state id is stale or missing', async () => { + const screensDir = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-deck-review-missing-')); + const html = fs.readFileSync(FIXTURE, 'utf8'); + const priorFilter = process.env.ARTIFACTURE_DECK_REVIEW_STATES; + process.env.ARTIFACTURE_DECK_REVIEW_STATES = 'slide-two--base,missing--state'; + const ctx = { + filePath: FIXTURE, + html, + profile: 'page', + preset: 'custom', + flags: { hasAnimations: false, hasMermaid: false }, + }; + + try { + await assert.rejects( + runBrowserStage(ctx, { screensDir, profile: 'page' }), + /did not capture requested states: missing--state/, + ); + } finally { + if (priorFilter === undefined) delete process.env.ARTIFACTURE_DECK_REVIEW_STATES; + else process.env.ARTIFACTURE_DECK_REVIEW_STATES = priorFilter; + fs.rmSync(screensDir, { recursive: true, force: true }); + } +}); + +test('bounded recapture fails closed when a requested state has no paired context', async () => { + const screensDir = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-deck-review-unpaired-')); + const html = fs.readFileSync(FIXTURE, 'utf8'); + const priorFilter = process.env.ARTIFACTURE_DECK_REVIEW_STATES; + process.env.ARTIFACTURE_DECK_REVIEW_STATES = 'slide-two--base'; + const ctx = { + filePath: FIXTURE, + html, + profile: 'page', + preset: 'custom', + flags: { hasAnimations: false, hasMermaid: false }, + }; + + try { + await assert.rejects( + runBrowserStage(ctx, { screensDir, profile: 'page' }), + /requires paired evidence for states: slide-two--base/, + ); + } finally { + if (priorFilter === undefined) delete process.env.ARTIFACTURE_DECK_REVIEW_STATES; + else process.env.ARTIFACTURE_DECK_REVIEW_STATES = priorFilter; + fs.rmSync(screensDir, { recursive: true, force: true }); + } +}); + +test('bounded recapture rejects a drill or progressive state without its base', async () => { + const screensDir = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-deck-review-orphan-')); + const html = fs.readFileSync(FIXTURE, 'utf8'); + const priorFilter = process.env.ARTIFACTURE_DECK_REVIEW_STATES; + process.env.ARTIFACTURE_DECK_REVIEW_STATES = 'slide-two--drill--state-1-progressive-evidence'; + const ctx = { + filePath: FIXTURE, + html, + profile: 'page', + preset: 'custom', + flags: { hasAnimations: false, hasMermaid: false }, + }; + + try { + await assert.rejects( + runBrowserStage(ctx, { screensDir, profile: 'page' }), + /requires two distinct states in every group/, + ); + } finally { + if (priorFilter === undefined) delete process.env.ARTIFACTURE_DECK_REVIEW_STATES; + else process.env.ARTIFACTURE_DECK_REVIEW_STATES = priorFilter; + fs.rmSync(screensDir, { recursive: true, force: true }); + } +}); + +test('browser stage can recapture one exact affected state with its base context', async () => { + const screensDir = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-deck-review-filter-')); + const html = fs.readFileSync(FIXTURE, 'utf8'); + const priorFilter = process.env.ARTIFACTURE_DECK_REVIEW_STATES; + process.env.ARTIFACTURE_DECK_REVIEW_STATES = 'slide-two--base,slide-two--drill--state-1-progressive-evidence'; + const ctx = { + filePath: FIXTURE, + html, + profile: 'page', + preset: 'custom', + flags: { hasAnimations: false, hasMermaid: false }, + }; + + try { + await runBrowserStage(ctx, { screensDir, profile: 'page' }); + const manifests = fs.readdirSync(screensDir) + .filter((name) => /^deck-review-.*\.json$/.test(name)) + .map((name) => JSON.parse(fs.readFileSync(path.join(screensDir, name), 'utf8'))); + assert.equal(manifests.length, 2); + assert.ok(manifests.every((manifest) => + manifest.units.length === 2 + && manifest.units[0].state_id === 'slide-two--base' + && manifest.units[1].state_id === 'slide-two--drill--state-1-progressive-evidence' + && manifest.review_groups.length === 1 + && manifest.review_groups[0].state_ids.length === 2)); + } finally { + if (priorFilter === undefined) delete process.env.ARTIFACTURE_DECK_REVIEW_STATES; + else process.env.ARTIFACTURE_DECK_REVIEW_STATES = priorFilter; + fs.rmSync(screensDir, { recursive: true, force: true }); + } +}); diff --git a/evals/fixtures/deck-review/presentation-states.html b/evals/fixtures/deck-review/presentation-states.html new file mode 100644 index 0000000..6ab1343 --- /dev/null +++ b/evals/fixtures/deck-review/presentation-states.html @@ -0,0 +1,62 @@ + + + + + + + + +
+
+
+

One base and one drill

+ +
+
+
+ + + diff --git a/evals/visual-model-policy/README.md b/evals/visual-model-policy/README.md index 93e8ffb..449e9b6 100644 --- a/evals/visual-model-policy/README.md +++ b/evals/visual-model-policy/README.md @@ -33,11 +33,13 @@ A cheaper larger model does not displace a qualified smaller model. Candidate ## Corpus -`corpus.json` defines a deliberately compact seed corpus: 36 paired scenarios -and 72 label-blind states across the owned families: +`corpus.json` defines a deliberately compact seed corpus: 42 paired scenarios +and 84 label-blind states across the owned families: - layout, including clipping, focal crowding, authored versus accidental dead space, and repeated-track symmetry; +- deck review, including rendered example truth, annotation mapping, reading + path density, adjacent-slide variety, and click-in continuity; - diagram fidelity; - aesthetic and preset fidelity; - operating-model fidelity; and diff --git a/evals/visual-model-policy/corpus.json b/evals/visual-model-policy/corpus.json index 46c1167..97dcf6a 100644 --- a/evals/visual-model-policy/corpus.json +++ b/evals/visual-model-policy/corpus.json @@ -252,6 +252,191 @@ "height": 720 } }, + { + "id": "deck-review", + "route_key": "deck-review", + "rubric": "../../plugins/visual-explainer/scripts/verify/rubrics/pass-deck-review.md", + "label_review": { + "status": "pending-human-review", + "reviewer": null, + "reviewed_at": null, + "notes": "Synthetic seed evidence only. A named human must inspect every rendered fire/clean pair before this family can contribute to qualification; production-shaped state groups are also required before graduation." + }, + "pairs": [ + { + "id": "rendered-example-truth", + "criterion_id": "deck-example-rendered-truth", + "region": { + "id": "example-pair", + "label": "Positive and negative example pair" + }, + "fire": { + "title": "Warning decoration is the only visible failure", + "visible_text": "Good and bad examples must earn their labels", + "truth_excerpt": "The negative example is supposed to demonstrate broken symmetry.", + "adjudication_notes": "Both examples have equal tracks and baselines; only the warning rule and caption claim the right example is bad.", + "render": { + "template": "deck", + "variant": "warning-only-example" + } + }, + "clean": { + "title": "Negative example visibly breaks symmetry", + "visible_text": "Good and bad examples must earn their labels", + "truth_excerpt": "The negative example is supposed to demonstrate broken symmetry.", + "adjudication_notes": "The negative example has unequal tracks and a displaced baseline before its caption is read.", + "render": { + "template": "deck", + "variant": "honest-example" + } + } + }, + { + "id": "annotation-mapping", + "criterion_id": "deck-annotation-evidence-mapping", + "region": { + "id": "annotated-examples", + "label": "Examples and explanatory annotation" + }, + "fire": { + "title": "Annotation for B sits beneath A", + "visible_text": "Annotations should map directly to evidence", + "truth_excerpt": "The annotation describes Example B.", + "adjudication_notes": "The annotation is aligned under Example A without a connector, so proximity assigns it to the wrong evidence.", + "render": { + "template": "deck", + "variant": "detached-annotation" + } + }, + "clean": { + "title": "Annotation shares Example B's column", + "visible_text": "Annotations should map directly to evidence", + "truth_excerpt": "The annotation describes Example B.", + "adjudication_notes": "The annotation is directly aligned with the evidence it names.", + "render": { + "template": "deck", + "variant": "aligned-annotation" + } + } + }, + { + "id": "reading-path-density", + "criterion_id": "deck-reading-path-density", + "region": { + "id": "slide-body", + "label": "Slide body and explanatory regions" + }, + "fire": { + "title": "Three regions repeat one conclusion", + "visible_text": "One claim should have one clear reading path", + "truth_excerpt": "The slide only needs to explain one bounded decision.", + "adjudication_notes": "Explanation, checklist, and return contract repeat the same conclusion with equal visual weight.", + "render": { + "template": "deck", + "variant": "overexplained-reading-path" + } + }, + "clean": { + "title": "One decision leads and evidence supports it", + "visible_text": "One claim should have one clear reading path", + "truth_excerpt": "The slide only needs to explain one bounded decision.", + "adjudication_notes": "The decision is dominant and the smaller evidence region advances rather than repeats it.", + "render": { + "template": "deck", + "variant": "distilled-reading-path" + } + } + }, + { + "id": "adjacent-slide-variety", + "criterion_id": "deck-structural-variety", + "region": { + "id": "slide-strip", + "label": "Three adjacent slide frames" + }, + "fire": { + "title": "Three lessons reuse the same card pair", + "visible_text": "Transfer should survive different structures", + "truth_excerpt": "The adjacent slides teach three different review relationships.", + "adjudication_notes": "All three frames use the same equal good/bad card pair, making distinct relationships look mechanically identical.", + "render": { + "template": "deck", + "variant": "repeated-layouts" + } + }, + "clean": { + "title": "Adjacent lessons use fitting structures", + "visible_text": "Transfer should survive different structures", + "truth_excerpt": "The adjacent slides teach comparison, sequence, and primary-versus-supporting hierarchy.", + "adjudication_notes": "The frames vary their structure because their relationships differ while retaining one visual system.", + "render": { + "template": "deck", + "variant": "varied-layouts" + } + } + }, + { + "id": "click-in-continuity", + "criterion_id": "deck-click-in-continuity", + "region": { + "id": "click-in-frame", + "label": "Base evidence and open click-in" + }, + "fire": { + "title": "Click-in obscures context and clips its last line", + "visible_text": "Click-ins need their own full-frame review", + "truth_excerpt": "The click-in should add evidence while preserving context and dismissal controls.", + "adjudication_notes": "The oversized overlay covers the base evidence and trigger while its final line escapes the visible frame.", + "render": { + "template": "deck", + "variant": "overlay-collision" + } + }, + "clean": { + "title": "Click-in uses reserved space and stays dismissible", + "visible_text": "Click-ins need their own full-frame review", + "truth_excerpt": "The click-in should add evidence while preserving context and dismissal controls.", + "adjudication_notes": "The overlay occupies a separate lane, keeps the base anchor visible, and contains its close control and copy.", + "render": { + "template": "deck", + "variant": "clear-click-in" + } + } + }, + { + "id": "positive-example-integrity", + "criterion_id": "deck-example-rendered-truth", + "region": { + "id": "positive-example", + "label": "Example labelled as a passing repeated-track layout" + }, + "fire": { + "title": "Pass example has an underweighted middle track", + "visible_text": "Input · Router · Output should read as equal peers", + "truth_excerpt": "The example is labelled positive because all three regions are direct peers.", + "adjudication_notes": "The narrow middle track visibly contradicts the example's positive label and peer relationship.", + "render": { + "template": "tracks", + "variant": "missing-weight" + } + }, + "clean": { + "title": "Pass example gives direct peers equal tracks", + "visible_text": "Input · Router · Output should read as equal peers", + "truth_excerpt": "The example is labelled positive because all three regions are direct peers.", + "adjudication_notes": "The three direct peers have equal widths and aligned baselines, so the positive label is visually supported.", + "render": { + "template": "tracks", + "variant": "equal-peers" + } + } + } + ], + "viewport": { + "width": 1280, + "height": 720 + } + }, { "id": "diagram", "route_key": "diagram", diff --git a/evals/visual-model-policy/corpus.mjs b/evals/visual-model-policy/corpus.mjs index c67768c..d9bd7fd 100644 --- a/evals/visual-model-policy/corpus.mjs +++ b/evals/visual-model-policy/corpus.mjs @@ -20,6 +20,12 @@ export const RENDER_VARIANTS = Object.freeze(Object.fromEntries( 'diagram-space': ['collapsed-column', 'editorial-space', 'intentional-breathing', 'tiny-unbalanced'], tracks: ['baseline-drift', 'editorial-60-40', 'equal-peers', 'hero-span', 'masonry', 'missing-weight', 'unequal-peers'], mobile: ['compact-controls', 'crowded-controls'], + deck: [ + 'aligned-annotation', 'clear-click-in', 'detached-annotation', + 'distilled-reading-path', 'honest-example', 'overlay-collision', + 'overexplained-reading-path', 'repeated-layouts', 'varied-layouts', + 'warning-only-example', + ], diagram: [ 'ambiguous-relations', 'annotation-no-legend', 'coherent-flow', 'complete-legend', 'decorated-list', 'dishonest-bars', 'dishonest-timeline', 'dual-focal', diff --git a/evals/visual-model-policy/corpus.test.mjs b/evals/visual-model-policy/corpus.test.mjs index c09cd3b..7a55bdf 100644 --- a/evals/visual-model-policy/corpus.test.mjs +++ b/evals/visual-model-policy/corpus.test.mjs @@ -9,6 +9,7 @@ import { const REQUIRED_FAMILIES = [ 'layout', + 'deck-review', 'diagram', 'aesthetic', 'operating-model', @@ -58,7 +59,7 @@ test('layout corpus covers clipping, crowding, dead space, and symmetry', async test('every case exposes stable evidence identity, a named region, and adjudication notes', async () => { const cases = expandCorpus(await loadCorpus()); - assert.equal(cases.length, 72); + assert.equal(cases.length, 84); for (const entry of cases) { assert.match(entry.case_id, /^[a-z0-9][a-z0-9:-]+$/); assert.match(entry.state_id, /^[a-z0-9][a-z0-9:-]+$/); diff --git a/evals/visual-model-policy/render-corpus.mjs b/evals/visual-model-policy/render-corpus.mjs index b28f95b..53d3e5a 100644 --- a/evals/visual-model-policy/render-corpus.mjs +++ b/evals/visual-model-policy/render-corpus.mjs @@ -221,6 +221,7 @@ function renderTemplate(template, variant, evalCase) {
Policy 02
`; } + if (template === 'deck') return renderDeck(variant, evalCase); if (template === 'diagram') return renderDiagram(variant, evalCase); if (template === 'preset') return renderPreset(variant, evalCase); if (template === 'operating') return renderOperating(variant, evalCase); @@ -228,6 +229,58 @@ function renderTemplate(template, variant, evalCase) { throw new Error(`unsupported render template ${template}`); } +function renderDeck(variant, evalCase) { + const regionId = evalCase.regions[0].id; + const title = `
Deck review · rendered truth

${escapeHtml(evalCase.visible_text)}

`; + if (['warning-only-example', 'honest-example'].includes(variant)) { + const honest = variant === 'honest-example'; + return `${title}
+
GOOD
${['Plan','Visits','Goals'].map((label) => `
${label}
`).join('')}
PASS · equal tracks
+
BAD
${['Plan','Visits','Goals'].map((label, index) => `
${label}
`).join('')}
${honest ? '' : '
'}
FAIL · broken symmetry
+
`; + } + if (['detached-annotation', 'aligned-annotation'].includes(variant)) { + const aligned = variant === 'aligned-annotation'; + return `${title}
+
Example A
Evidence region
+
Example B
Evidence region
+ +
`; + } + if (['overexplained-reading-path', 'distilled-reading-path'].includes(variant)) { + const distilled = variant === 'distilled-reading-path'; + return `${title}
+

${distilled ? 'One visible decision' : 'Explanation'}

${distilled ? 'The examples and one bounded check carry the claim.' : 'The same conclusion is repeated here in full.'}

+

${distilled ? 'Evidence' : 'Checklist'}

${distilled ? 'Two aligned examples.' : 'The same conclusion is repeated here again.'}

+ ${distilled ? '' : '

Return contract

The same conclusion is repeated a third time.

'} +
`; + } + if (['repeated-layouts', 'varied-layouts'].includes(variant)) { + const varied = variant === 'varied-layouts'; + const frames = varied + ? [ + '
BeforeAfter
', + '
010203
', + '
PrimaryRail
', + ] + : Array.from({ length: 3 }, () => '
GoodBad
'); + return `${title}
${frames.map((frame, index) => `
SLIDE ${index + 1}
${frame}
`).join('')}
`; + } + if (['overlay-collision', 'clear-click-in'].includes(variant)) { + const clear = variant === 'clear-click-in'; + return `${title}
+
Base evidence

The visual anchor remains available.

+ + +
`; + } + throw new Error(`unsupported deck render variant ${variant}`); +} + function renderDiagram(variant, evalCase) { const regionId = evalCase.regions[0].id; const timeline = ['dishonest-timeline', 'timeline-break'].includes(variant); diff --git a/plugins/visual-explainer/.claude/agents/ve-verifier-deck-review.md b/plugins/visual-explainer/.claude/agents/ve-verifier-deck-review.md new file mode 100644 index 0000000..54a6faa --- /dev/null +++ b/plugins/visual-explainer/.claude/agents/ve-verifier-deck-review.md @@ -0,0 +1,37 @@ +--- +name: ve-verifier-deck-review +description: Judge rendered slide truth, annotation mapping, reading path, structural variety, and every click-in/progressive state. Returns only verdict JSON. +tools: Read +--- + +# ve-verifier-deck-review + +Run one audit pass. Do not fix files. + +## Read + +1. `plugins/visual-explainer/scripts/verify/rubrics/pass-deck-review.md` +2. One generated `deck-review-*.json` manifest. +3. Every `review_groups` entry in manifest order. Dispatch each group as its + exact two-frame evidence package; do not flatten the deck into one + oversized request. The route is ready only when its independently qualified + `batch_size` is `2`. Capture fails closed when a state has no valid base, + progression, drill, or adjacent-slide partner. +4. A visible title or one-sentence narrative job only when the frame does not + supply one. + +Treat the rendered pixels as truth. Complete every group so every base, drill, +and progressive state is reviewed and adjacent base-slide pairs are checked for +structural variety. Do not infer intent from implementation source or the +builder's explanation. + +## Output + +Return only one valid JSON object matching the rubric schema: + +```json +{"pass":true,"findings":[]} +``` + +Do not include prose, markdown, code fences, summaries, or recommendations +outside `findings`. diff --git a/plugins/visual-explainer/SKILL.md b/plugins/visual-explainer/SKILL.md index 9169f88..c30eae1 100644 --- a/plugins/visual-explainer/SKILL.md +++ b/plugins/visual-explainer/SKILL.md @@ -110,7 +110,7 @@ Clarify: `./references/clarify.md`. Use components/tokens, not hand CSS/coords. ## 6. Verify -Read `./references/verification.md`, `./references/delegated-skills.md`, and `./references/model-routing.md`; run `node {{skill_dir}}/scripts/verify/ve-verify.mjs --json --screens ` (in this repo, `{{skill_dir}}` = `plugins/visual-explainer`); run only the routed Artifacture passes and delegated Impeccable/Unslop checks. The main thread orchestrates and summarizes; it does not perform screenshot judgment. Dispatch each Artifacture-owned visual pass to the smallest eval-qualified model and exact batch size in the generated policy. If no qualified route can run, skip with `no-eval-qualified-model` instead of silently using the host/frontier model. Report the artifact path, report JSON, actual model/batch per pass, per-pass pass/fail, and an explicit disclosure if anything could not be verified. +Read `./references/verification.md`, `./references/delegated-skills.md`, and `./references/model-routing.md`; run `node {{skill_dir}}/scripts/verify/ve-verify.mjs --json --screens ` (in this repo, `{{skill_dir}}` = `plugins/visual-explainer`); run only the routed Artifacture passes and delegated Impeccable/Unslop checks. For a slide or fixed-stage presentation, consume the generated deck-review manifest and review every base, drill, and progressive state; never substitute the opening screenshot for the complete state set. If `P-deck-review` fails, read `./references/deck-visual-review-repair.md` and use that bounded repair operator on only the grounded findings before the final complete-deck recapture. The main thread orchestrates and summarizes; it does not perform screenshot judgment. Dispatch each Artifacture-owned visual pass to the smallest eval-qualified model and exact batch size in the generated policy. If no qualified route can run, skip with `no-eval-qualified-model` instead of silently using the host/frontier model. Report the artifact path, report JSON, actual model/batch per pass, per-pass pass/fail, and an explicit disclosure if anything could not be verified. ## Design-craft delegation diff --git a/plugins/visual-explainer/references/deck-visual-review-repair.md b/plugins/visual-explainer/references/deck-visual-review-repair.md new file mode 100644 index 0000000..04a5087 --- /dev/null +++ b/plugins/visual-explainer/references/deck-visual-review-repair.md @@ -0,0 +1,50 @@ +# Deck visual review repair + +Use this repair operator only after `P-deck-review` returns grounded findings. +The audit verdict defines the allowed repair scope. + +## Establish the repair set + +Read the affected slide source, the deck build command, the failing state ids, +and their manifest entries. Include: + +- every named failing base, drill, or progressive state; +- the related base frame for any failing interactive state; +- every state reachable before the failing state when progression is ordered; +- every importing deck when a shared visual component must change. + +Use `ARTIFACTURE_DECK_REVIEW_STATES=` with +`ve-verify` to recapture only an affected set. Omit it for the final complete +deck verification. + +## Repair rendered contradictions first + +Treat pixels as truth: + +- Positive examples must be visibly correct. +- Negative examples must fail through real geometry, hierarchy, state, or + behavior—not through a warning label or decorative mark alone. +- Put each annotation in the same row, column, or connected region as its + evidence. +- Remove repeated explanation before adding more structure. +- Vary the domain or composition when adjacent examples claim a transferable + lesson but reuse one tired fixture. +- Give click-ins reserved space, a clear dismissal path, and a dedicated + full-frame review. + +Preserve the slide's narrative job and the deck's established visual system. +Prefer the smallest structural repair. Do not turn an audit finding into a +general redesign. + +## Run the bounded loop + +1. Edit the authored MDX/TSX source, never exported HTML. +2. Export the deck. +3. Recapture the repair set with `ARTIFACTURE_DECK_REVIEW_STATES`. +4. Inspect every captured frame and run deterministic checks. +5. Rerun `P-deck-review` on the repair set. +6. Repeat at most twice. +7. Clear the filter and run one complete deck capture before delivery. + +If a shared component changed, rerender and inspect every consuming deck before +completion. diff --git a/plugins/visual-explainer/references/verification.md b/plugins/visual-explainer/references/verification.md index 51edf5a..27c05c9 100644 --- a/plugins/visual-explainer/references/verification.md +++ b/plugins/visual-explainer/references/verification.md @@ -74,10 +74,11 @@ judgment may not. 2. Build the P-completeness source-vs-render inventory from the source material and the rendered artifact: headings, bullets, table rows, cards, collapsible details, footnotes, and demo-frame summaries. Do not use screenshots for this pass. 3. Capture P-diagram inputs one figure at a time. Scroll to each element carrying `data-diagram-role` or `.mermaid`; screenshot that element's bounding region; extract its visible labels; write a one-line content brief for the figure. -4. For P-operating-model, build a review map with one row per slide or coherent long-form section: stable unit id, unit type (`slide` or `section`), visible title, one-sentence narrative job, and screenshot path. If the source has no narrative job, use the visible claim and mark the route low-confidence. Do not use implementation source to infer intent. -5. Use candidate element lists from each deterministic check's `evidence` and `where` fields when a pass asks for candidate extracts. -6. For `impeccable:critique`, capture only the candidate screenshots and locations named by the report. For `unslop:cleanup-report`, supply only the excluded-filtered prose extract. -7. For `artifacture:slop-gap`, capture only the explicitly nominated region, its visible text, and the smallest source/truth excerpt needed to judge sequence, state/confidence, or provenance. +4. For P-deck-review, use the generated `deck-review-*.json` sidecar manifest in the screens directory. Each base slide, visible drill target, and custom progressive state receives a stable state id at the verifier's contracted desktop viewport. Run the ordered `review_groups`: state-continuity groups pair a base with each drill or progressive state, while adjacent-slide-variety groups pair neighboring base slides. Every group contains exactly two production frames. P-deck-review requires an independently qualified batch size of 2; a batch-1 route is unavailable, and capture fails closed when a lone state has no valid partner. Do not substitute the opening frame for the complete state set. +5. For P-operating-model, build a review map with one row per slide or coherent long-form section: stable unit id, unit type (`slide` or `section`), visible title, one-sentence narrative job, and screenshot path. If the source has no narrative job, use the visible claim and mark the route low-confidence. Do not use implementation source to infer intent. +6. Use candidate element lists from each deterministic check's `evidence` and `where` fields when a pass asks for candidate extracts. +7. For `impeccable:critique`, capture only the candidate screenshots and locations named by the report. For `unslop:cleanup-report`, supply only the excluded-filtered prose extract. +8. For `artifacture:slop-gap`, capture only the explicitly nominated region, its visible text, and the smallest source/truth excerpt needed to judge sequence, state/confidence, or provenance. ## 2. Run LLM Passes @@ -149,6 +150,7 @@ Use these passes: | Pass | Run When | Inputs | Rubric | |---|---|---|---| | P-layout | `llm_passes_required` includes `hierarchy`, or any layout candidate exists | `report.json`; the 4 standard screenshots; candidate lists for clipping, fixed chrome, div-grid tables, repeated-track layouts, slide screenshots, demo frames when referenced | `{{skill_dir}}/scripts/verify/rubrics/pass-layout.md` | +| P-deck-review | `llm_passes_required` includes `deck-review` | one generated deck-review manifest; every ordered state-continuity and adjacent-slide-variety group named by it; visible title or narrative job only when absent from the frame | `{{skill_dir}}/scripts/verify/rubrics/pass-deck-review.md` | | P-aesthetic | `llm_passes_required` includes any `aesthetic-*` token, a preset is detected or declared, or any preset candidate exists | `report.json`; active preset name; light/dark screenshots; candidate extracts named in the report | `{{skill_dir}}/scripts/verify/rubrics/pass-aesthetic.md` | | P-diagram | diagrams are present | `report.json`; one screenshot per figure; extracted diagram labels; one-line content brief for each figure | `{{skill_dir}}/scripts/verify/rubrics/pass-diagram.md` | | P-operating-model | `llm_passes_required` includes `operating-model` | one screenshot per review unit; review map with unit id, unit type, visible title, and one-sentence narrative job; only the brief excerpts needed to resolve intent | `{{skill_dir}}/scripts/verify/rubrics/pass-operating-model.md` | @@ -162,6 +164,15 @@ Use these passes: Ask only the applicable questions tagged in `pass-layout.md`: semantic table need, global hierarchy, visible text clipping, mobile fixed-chrome obstruction, slide focal clarity, repeated-track symmetry, repeated slide composition, and sparse diagram slide. Repeated-track symmetry is conditional: run it only when the artifact visibly establishes equivalent columns or rows. +### P-deck-review Questions + +Treat rendered frames as truth. Check whether labeled examples visibly earn +their labels, annotations map to their evidence, the reading path survives the +slide's density, adjacent slides demonstrate meaningful structural variety, and +every drill or progressive state remains clear and collision-free. Attribute +findings to exact state ids. Generic visual craft remains delegated to +Impeccable; prose remains delegated to Unslop. + ### P-aesthetic Questions Load only the active named-preset section in `pass-aesthetic.md`. Do not judge generic custom pages against a named preset; general visual craft and visual AI tells route to Impeccable. @@ -216,7 +227,13 @@ Inspect the exact exported PNG after every `poster export`. Check edge clipping, 2. If every pass returns `"pass": true`, continue to Step 4. 3. If any pass returns `"pass": false`, fix only the defects named in `findings`. 4. Re-export from source. Never hand-edit generated HTML when an MDX/TSX source exists. -5. Rerun `ve-verify`. +5. Rerun `ve-verify`. For deck-review findings, first read + `./deck-visual-review-repair.md`, then set + `ARTIFACTURE_DECK_REVIEW_STATES` to the comma-separated affected state ids + and related base/progression states. This recaptures only the repair set and + fails if any requested state id is stale or missing; a repaired base frame + does not prove its related states are repaired. Clear the + variable and run a complete deck capture before delivery. 6. Rerun only the affected LLM passes. 7. Repeat this LLM fix loop at most 2 times. If any pass still fails, stop and disclose the unresolved findings. diff --git a/plugins/visual-explainer/scripts/verify/lib/browser.mjs b/plugins/visual-explainer/scripts/verify/lib/browser.mjs index 381c027..6b80891 100644 --- a/plugins/visual-explainer/scripts/verify/lib/browser.mjs +++ b/plugins/visual-explainer/scripts/verify/lib/browser.mjs @@ -1,10 +1,12 @@ -import { access, mkdir, readFile, readdir } from 'node:fs/promises'; +import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { homedir, tmpdir } from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { chromium } from 'playwright-core'; const DEFAULT_SCREENS_DIR = path.join(tmpdir(), 've-verify-screens'); +const MAX_DECK_REVIEW_SLIDES = 200; +const DECK_REVIEW_SETTLE_MS = 240; export async function runBrowserStage(ctx, options = {}) { const profile = options.profile || ctx.profile || 'page'; @@ -264,6 +266,13 @@ async function executeRun(browser, ctx, runMeta, screensDir) { const last = tags[tags.length - 1]; if (last && last.textContent && last.textContent.includes('.ve-review-panel { visibility: hidden')) last.remove(); }); + await captureDeckReviewSet({ + page, + ctx, + runMeta, + screensDir, + suffix, + }); return { viewport: runMeta.viewport, @@ -1499,3 +1508,354 @@ const BROWSER_METRIC_SOURCE = [ reelTypographyMetrics, mermaidMetrics ].map((fn) => fn.toString()).join('\n'); + +function deckReviewCaptureMode({ + html = '', + profile = 'page', + width = 0, + reducedMotion = false, +} = {}) { + if (reducedMotion || width < 1000) return null; + if (/data-ve-presentation/i.test(html)) return 'presentation'; + if (profile === 'slides') return 'scroll-deck'; + return null; +} + +function sanitizeReviewId(value) { + const sanitized = String(value ?? '') + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); + return sanitized || 'untitled'; +} + +function reviewStateId(slideId, kind, detail) { + const parts = [sanitizeReviewId(slideId), sanitizeReviewId(kind)]; + if (detail !== undefined && detail !== null && detail !== '') { + parts.push(sanitizeReviewId(detail)); + } + return parts.join('--'); +} + +function parseReviewStateFilter(value = '') { + const stateIds = String(value) + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean); + return stateIds.length ? new Set(stateIds) : null; +} + +async function captureDeckReviewSet({ + page, + ctx, + runMeta, + screensDir, + suffix, +}) { + const mode = deckReviewCaptureMode({ + html: ctx.html, + profile: runMeta.profile, + width: runMeta.width, + reducedMotion: runMeta.reducedMotion, + }); + if (!mode) return null; + + const stateFilter = parseReviewStateFilter(process.env.ARTIFACTURE_DECK_REVIEW_STATES); + const units = mode === 'presentation' + ? await captureReviewPresentation(page, screensDir, suffix, stateFilter) + : await captureReviewScrollDeck(page, screensDir, suffix, stateFilter); + assertCompleteReviewFilter(stateFilter, units); + const manifestPath = path.join(screensDir, `deck-review-${suffix}.json`); + const reviewGroups = buildDeckReviewGroups(units); + assertCompleteDeckReviewGroups(units, reviewGroups); + const manifest = { + schema_version: 1, + kind: 'deck-review-set', + mode, + viewport: { width: runMeta.width, height: runMeta.height }, + scheme: runMeta.scheme, + state_filter: stateFilter ? [...stateFilter] : null, + units, + review_groups: reviewGroups, + }; + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + return { manifestPath, mode, units }; +} + +function assertCompleteReviewFilter(stateFilter, units) { + if (!stateFilter) return; + const captured = new Set(units.map((unit) => unit.state_id)); + const missing = [...stateFilter].filter((stateId) => !captured.has(stateId)); + if (missing.length) { + throw new Error(`deck review did not capture requested states: ${missing.join(', ')}`); + } +} + +function buildDeckReviewGroups(units) { + const bySlide = new Map(); + for (const unit of units) { + const group = bySlide.get(unit.slide_id) || []; + group.push(unit); + bySlide.set(unit.slide_id, group); + } + const groups = []; + const bases = []; + for (const [slideId, slideUnits] of bySlide) { + const base = slideUnits.find((unit) => unit.state_kind === 'base'); + if (base) bases.push(base); + const variants = slideUnits.filter((unit) => unit !== base); + for (const variant of variants) { + groups.push({ + group_id: `${sanitizeReviewId(slideId)}--${sanitizeReviewId(variant.state_id)}-context`, + purpose: 'state-continuity', + state_ids: base ? [base.state_id, variant.state_id] : [variant.state_id], + }); + } + } + for (let index = 0; index < bases.length - 1; index += 1) { + groups.push({ + group_id: `${sanitizeReviewId(bases[index].slide_id)}--${sanitizeReviewId(bases[index + 1].slide_id)}--sequence`, + purpose: 'adjacent-slide-variety', + state_ids: [bases[index].state_id, bases[index + 1].state_id], + }); + } + return groups; +} + +function assertCompleteDeckReviewGroups(units, groups) { + const malformed = groups.filter((group) => + group.state_ids.length !== 2 || new Set(group.state_ids).size !== 2); + if (malformed.length) { + throw new Error(`deck review requires two distinct states in every group: ${malformed.map((group) => group.group_id).join(', ')}`); + } + const grouped = new Set(groups.flatMap((group) => group.state_ids)); + const unpaired = units.filter((unit) => !grouped.has(unit.state_id)).map((unit) => unit.state_id); + if (unpaired.length) { + throw new Error(`deck review requires paired evidence for states: ${unpaired.join(', ')}`); + } +} + +async function captureReviewPresentation(page, screensDir, suffix, stateFilter) { + await page.keyboard.press('Home'); + await waitForDeckReviewPaint(page); + const units = []; + const seenSlides = new Set(); + + for (let ordinal = 0; ordinal < MAX_DECK_REVIEW_SLIDES; ordinal += 1) { + const meta = await deckReviewPresentationMeta(page); + if (!meta || seenSlides.has(meta.index)) break; + seenSlides.add(meta.index); + const slideId = meta.slideId || `slide-${meta.index + 1}`; + pushDeckReviewUnit(units, await captureDeckReviewViewport(page, screensDir, suffix, { + slideId, + slideIndex: meta.index, + title: meta.title, + stateKind: 'base', + stateIndex: 0, + }, stateFilter)); + await captureDeckReviewDrills(page, screensDir, suffix, units, { + slideId, + slideIndex: meta.index, + title: meta.title, + parentStateIndex: meta.customStateIndex, + includeParentState: meta.customStateCount > 1, + drills: meta.drills, + stateFilter, + }); + + if (meta.customStateCount > 1) { + for (let stateIndex = meta.customStateIndex + 1; stateIndex < meta.customStateCount; stateIndex += 1) { + await page.keyboard.press('ArrowDown'); + await page.waitForFunction( + (expected) => Number(document.querySelector('[data-presentation-state-nav="true"]')?.getAttribute('data-presentation-state-index')) === expected, + stateIndex, + { timeout: 3000 }, + ); + await waitForDeckReviewPaint(page); + pushDeckReviewUnit(units, await captureDeckReviewViewport(page, screensDir, suffix, { + slideId, + slideIndex: meta.index, + title: meta.title, + stateKind: 'state', + stateIndex, + stateDetail: stateIndex, + }, stateFilter)); + const stateMeta = await deckReviewPresentationMeta(page); + await captureDeckReviewDrills(page, screensDir, suffix, units, { + slideId, + slideIndex: meta.index, + title: meta.title, + parentStateIndex: stateIndex, + includeParentState: true, + drills: stateMeta?.drills || [], + stateFilter, + }); + } + } + + await page.keyboard.press('ArrowRight'); + const changed = await page.waitForFunction( + (previous) => Number(document.querySelector('[data-slide-index]')?.getAttribute('data-slide-index')) !== previous, + meta.index, + { timeout: 900 }, + ).then(() => true).catch(() => false); + if (!changed) break; + await waitForDeckReviewPaint(page); + } + return units; +} + +async function captureReviewScrollDeck(page, screensDir, suffix, stateFilter) { + const selector = await page.evaluate(() => { + if (document.querySelectorAll('section.slide').length) return 'section.slide'; + if (document.querySelectorAll('[data-ve-deck] > section').length) return '[data-ve-deck] > section'; + if (document.querySelectorAll('[data-slide-id]').length) return '[data-slide-id]'; + return ''; + }); + if (!selector) return []; + const slides = page.locator(selector); + const count = Math.min(await slides.count(), MAX_DECK_REVIEW_SLIDES); + const units = []; + + for (let index = 0; index < count; index += 1) { + const slide = slides.nth(index); + await slide.scrollIntoViewIfNeeded(); + await waitForDeckReviewPaint(page); + const meta = await slide.evaluate((element, slideIndex) => ({ + slideId: element.getAttribute('data-slide-id') || element.id || `slide-${slideIndex + 1}`, + title: (element.querySelector('h1,h2,h3,[data-slide-title]')?.textContent || '').replace(/\s+/g, ' ').trim(), + }), index); + pushDeckReviewUnit(units, await captureDeckReviewViewport(page, screensDir, suffix, { + slideId: meta.slideId, + slideIndex: index, + title: meta.title, + stateKind: 'base', + stateIndex: 0, + }, stateFilter)); + const drills = await visibleDeckReviewDrills(slide); + await captureDeckReviewDrills(page, screensDir, suffix, units, { + slideId: meta.slideId, + slideIndex: index, + title: meta.title, + parentStateIndex: 0, + includeParentState: false, + drills, + stateFilter, + scope: slide, + }); + } + return units; +} + +async function deckReviewPresentationMeta(page) { + return page.evaluate(() => { + const stage = document.querySelector('[data-slide-index]'); + if (!stage) return null; + const custom = stage.querySelector('[data-presentation-state-nav="true"]'); + const title = stage.querySelector('h1,h2,h3,[data-slide-title]')?.textContent || ''; + return { + index: Number(stage.getAttribute('data-slide-index')) || 0, + slideId: stage.getAttribute('data-slide-id') || window.location.hash.replace(/^#/, ''), + title: title.replace(/\s+/g, ' ').trim(), + customStateIndex: Number(custom?.getAttribute('data-presentation-state-index')) || 0, + customStateCount: Number(custom?.getAttribute('data-presentation-state-count')) || 0, + drills: [...stage.querySelectorAll('[data-drill-target]')] + .filter((element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden'; + }) + .map((element) => ({ + id: element.getAttribute('data-drill-target') || '', + label: (element.getAttribute('aria-label') || element.textContent || '').replace(/\s+/g, ' ').trim(), + })) + .filter((entry) => entry.id), + }; + }); +} + +async function visibleDeckReviewDrills(scope) { + return scope.locator('[data-drill-target]').evaluateAll((elements) => + elements + .filter((element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden'; + }) + .map((element) => ({ + id: element.getAttribute('data-drill-target') || '', + label: (element.getAttribute('aria-label') || element.textContent || '').replace(/\s+/g, ' ').trim(), + })) + .filter((entry) => entry.id), + ); +} + +async function captureDeckReviewDrills(page, screensDir, suffix, units, { + slideId, + slideIndex, + title, + parentStateIndex, + includeParentState, + drills, + stateFilter, + scope = page, +}) { + for (const drill of drills) { + const detail = includeParentState ? `state-${parentStateIndex}-${drill.id}` : drill.id; + const stateId = reviewStateId(slideId, 'drill', detail); + if (stateFilter && !stateFilter.has(stateId)) continue; + await scope.locator(`[data-drill-target="${cssDeckReviewString(drill.id)}"]`).first().click(); + await page.waitForSelector('[data-drill-open]', { state: 'visible', timeout: 3000 }); + await waitForDeckReviewPaint(page); + pushDeckReviewUnit(units, await captureDeckReviewViewport(page, screensDir, suffix, { + slideId, + slideIndex, + title, + stateKind: 'drill', + stateIndex: parentStateIndex, + stateDetail: detail, + stateTitle: drill.label, + }, stateFilter)); + await closeDeckReviewDrill(page); + } +} + +async function captureDeckReviewViewport(page, screensDir, suffix, state, stateFilter) { + const stateId = reviewStateId(state.slideId, state.stateKind, state.stateDetail); + if (stateFilter && !stateFilter.has(stateId)) return null; + const screenshotPath = path.join(screensDir, `deck-review-${suffix}-${stateId}.png`); + await page.screenshot({ path: screenshotPath }); + return { + state_id: stateId, + slide_id: String(state.slideId), + slide_index: state.slideIndex, + slide_title: state.title || '', + state_kind: state.stateKind, + state_index: state.stateIndex ?? null, + state_title: state.stateTitle || '', + screenshot_path: screenshotPath, + }; +} + +function pushDeckReviewUnit(units, unit) { + if (unit) units.push(unit); +} + +async function closeDeckReviewDrill(page) { + const close = page.locator('[data-drill-close]').first(); + if (await close.count()) await close.click(); + else await page.keyboard.press('Escape'); + await page.waitForSelector('[data-drill-open]', { state: 'detached', timeout: 3000 }).catch(() => {}); + await waitForDeckReviewPaint(page); +} + +async function waitForDeckReviewPaint(page) { + await page.waitForTimeout(DECK_REVIEW_SETTLE_MS); + await page.evaluate(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))); +} + +function cssDeckReviewString(value) { + return String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +} diff --git a/plugins/visual-explainer/scripts/verify/lib/model-policy.mjs b/plugins/visual-explainer/scripts/verify/lib/model-policy.mjs index 22c45a8..0110e6a 100644 --- a/plugins/visual-explainer/scripts/verify/lib/model-policy.mjs +++ b/plugins/visual-explainer/scripts/verify/lib/model-policy.mjs @@ -58,6 +58,15 @@ export function buildLlmDispatchPlan(passes, resolved = resolveVisualModelPolicy policy_source: resolved.source, }; } + if (routeKey === 'deck-review' && Number(route.batch_size) !== 2) { + return { + pass, + owner: 'artifacture', + status: 'skipped', + reason: 'deck-review-requires-paired-evidence', + policy_source: resolved.source, + }; + } return { pass, owner: 'artifacture', diff --git a/plugins/visual-explainer/scripts/verify/lib/model-policy.test.mjs b/plugins/visual-explainer/scripts/verify/lib/model-policy.test.mjs index c812917..28317cb 100644 --- a/plugins/visual-explainer/scripts/verify/lib/model-policy.test.mjs +++ b/plugins/visual-explainer/scripts/verify/lib/model-policy.test.mjs @@ -7,6 +7,8 @@ import { buildLlmDispatchPlan, resolveVisualModelPolicy, } from './model-policy.mjs'; +import '../../../../../evals/deck-review-set.browser.test.mjs'; +import '../../../../../evals/deck-review-report.test.mjs'; function generatedPolicy(routes) { return { @@ -94,6 +96,33 @@ test('does not borrow layout qualification for an aesthetic pass', () => { assert.equal(entry.reason, 'no-eval-qualified-model'); }); +test('does not borrow layout qualification for deck review', () => { + const [entry] = buildLlmDispatchPlan( + ['deck-review'], + { source: '/policy.json', policy: generatedPolicy({ layout: measuredRoute() }) }, + ); + assert.equal(entry.status, 'skipped'); + assert.equal(entry.reason, 'no-eval-qualified-model'); +}); + +test('deck review refuses a route that was not qualified on paired evidence', () => { + const [entry] = buildLlmDispatchPlan( + ['deck-review'], + { source: '/policy.json', policy: generatedPolicy({ 'deck-review': measuredRoute({ batch_size: 1 }) }) }, + ); + assert.equal(entry.status, 'skipped'); + assert.equal(entry.reason, 'deck-review-requires-paired-evidence'); +}); + +test('deck review accepts an independently qualified paired-evidence route', () => { + const [entry] = buildLlmDispatchPlan( + ['deck-review'], + { source: '/policy.json', policy: generatedPolicy({ 'deck-review': measuredRoute({ batch_size: 2 }) }) }, + ); + assert.equal(entry.status, 'ready'); + assert.equal(entry.batch_size, 2); +}); + test('rejects a hand-written route without selector provenance and measurements', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'artifacture-policy-')); const explicit = path.join(root, 'hand-written.json'); diff --git a/plugins/visual-explainer/scripts/verify/lib/report.mjs b/plugins/visual-explainer/scripts/verify/lib/report.mjs index bfdb771..c7e779d 100644 --- a/plugins/visual-explainer/scripts/verify/lib/report.mjs +++ b/plugins/visual-explainer/scripts/verify/lib/report.mjs @@ -131,6 +131,10 @@ function llmPassesFor(ctx, checks) { if (declaresCheck(ctx.html || '', ['artifacture:slop-gap'])) { required.add('artifacture:slop-gap'); } + const hasDeckReviewUnits = + ctx.profile === 'slides' || + /data-ve-presentation/i.test(ctx.html || ''); + if (hasDeckReviewUnits) required.add('deck-review'); const hasReviewUnits = ['slides', 'magazine'].includes(ctx.profile) || /data-ve-presentation/i.test(ctx.html) || diff --git a/plugins/visual-explainer/scripts/verify/rubric-criteria.json b/plugins/visual-explainer/scripts/verify/rubric-criteria.json index b5ab8ea..6a089d4 100644 --- a/plugins/visual-explainer/scripts/verify/rubric-criteria.json +++ b/plugins/visual-explainer/scripts/verify/rubric-criteria.json @@ -6,6 +6,31 @@ "pass": "layout", "rubric": "rubrics/pass-layout.md" }, + { + "id": "deck-example-rendered-truth", + "pass": "deck-review", + "rubric": "rubrics/pass-deck-review.md" + }, + { + "id": "deck-annotation-evidence-mapping", + "pass": "deck-review", + "rubric": "rubrics/pass-deck-review.md" + }, + { + "id": "deck-reading-path-density", + "pass": "deck-review", + "rubric": "rubrics/pass-deck-review.md" + }, + { + "id": "deck-structural-variety", + "pass": "deck-review", + "rubric": "rubrics/pass-deck-review.md" + }, + { + "id": "deck-click-in-continuity", + "pass": "deck-review", + "rubric": "rubrics/pass-deck-review.md" + }, { "id": "artifact-slop-false-sequence", "pass": "artifacture:slop-gap", diff --git a/plugins/visual-explainer/scripts/verify/rubrics/pass-deck-review.md b/plugins/visual-explainer/scripts/verify/rubrics/pass-deck-review.md new file mode 100644 index 0000000..b87109e --- /dev/null +++ b/plugins/visual-explainer/scripts/verify/rubrics/pass-deck-review.md @@ -0,0 +1,61 @@ +Inputs: +- The `deck-review-*.json` manifest for one scheme. +- One manifest `review_groups` entry at a time, using exactly the two + full-frame screenshots named by its `state_ids`. +- The slide's visible title or one-sentence narrative job when it is not visible in the frame. +- No implementation source and no builder explanation. + +Judge the rendered frames as truth. Complete every group in manifest order; +merge and deduplicate findings by `check_id`, `state_id`, and region. Labels, +captions, and source intent do not override contradictory pixels. + +Questions: +- [deck-example-rendered-truth] When a slide labels examples positive/good/pass + or negative/bad/fail, do the rendered examples visibly earn those labels? + A negative must fail through real geometry, containment, hierarchy, state, or + behavior—not through a decorative underline, warning color, or caption alone. +- [deck-annotation-evidence-mapping] Does every load-bearing annotation occupy + the same row, column, connected region, or unmistakable visual path as the + evidence it explains? +- [deck-reading-path-density] Is the intended first read obvious, with repeated + explanation and redundant interface anatomy removed? Do not flag expert + density when hierarchy remains clear and every region advances the claim. +- [deck-structural-variety] Across adjacent slides, do examples vary in domain, + layout, or symmetry when the lesson claims to transfer? Flag a sequence that + repeatedly uses the same fixture or silhouette when that repetition makes the + method look narrower or the presentation feel mechanically templated. +- [deck-click-in-continuity] For every drill or progressive state, does the new + content remain unclipped, collision-free, legible, and visually continuous + with the base state? The click-in must add useful detail without obscuring the + evidence or controls needed to understand or dismiss it. + +Failure threshold: +- Return a finding only for a visible contradiction, ambiguous mapping, + materially confusing reading path, credibility-reducing repetition, or + broken interactive state. +- Attribute every finding to one or more exact `state_id` values and visible + regions. +- Stay silent on generic typography, color taste, prose style, and visual AI + tells; those belong to Impeccable or Unslop. +- Stay silent when repeated structure is semantically meaningful, such as a + deliberate sequence, stable comparison matrix, or recurring navigation shell. + +Verdict JSON schema: +`{"pass":true,"findings":[{"check_id":"","state_id":"s03--drill--evidence","region":"right example lane","evidence":"...","fix":"..."}]}` + +Pass examples: +- A pass example has equal tracks and baselines; its paired fail example visibly + breaks those tracks before the viewer reads the caption. +- Annotation, example, and binary check form one aligned horizontal lane. +- Three adjacent slides teach containment, symmetry, and action priority with + materially different compositions. +- A click-in expands into reserved space, keeps its dismissal affordance clear, + and preserves the base slide's visual anchor. + +Fail examples: +- “Bad” differs from “good” only through a rust underline and label. +- A note describing the right-hand example sits under the left-hand example. +- The slide repeats the same explanation in a caption, checklist, and footer. +- Four adjacent slides reuse the same centered card pair despite claiming a + general review method. +- A drill sheet clips its last row or covers the trigger and close control.