From 24788447f24a007ecf528bfb3844b2d3efcd5d34 Mon Sep 17 00:00:00 2001 From: Alex Gemberg Date: Fri, 19 Dec 2025 09:08:16 +1100 Subject: [PATCH 1/2] tests: add JSON diff reports for sparse test snapshot failures --- Cargo.lock | 2 + Cargo.toml | 2 + sparse_strips/vello_sparse_tests/Cargo.toml | 2 + .../vello_sparse_tests/tests/util.rs | 97 ++++++++++++++++++- 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0671e2c4..f3e5bd346 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4090,6 +4090,8 @@ dependencies = [ "image", "oxipng", "pollster", + "serde", + "serde_json", "skrifa 0.37.0", "smallvec", "vello_api", diff --git a/Cargo.toml b/Cargo.toml index 3dad9c96d..ac201d0f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -154,3 +154,5 @@ console_log = "1.0" proc-macro2 = "1.0.95" syn = { version = "2.0.101", features = ["full", "extra-traits"] } quote = "1.0.40" +serde = { version = "1.0", default-features = false } +serde_json = "1.0" diff --git a/sparse_strips/vello_sparse_tests/Cargo.toml b/sparse_strips/vello_sparse_tests/Cargo.toml index aa6dbd555..aebfd59bb 100644 --- a/sparse_strips/vello_sparse_tests/Cargo.toml +++ b/sparse_strips/vello_sparse_tests/Cargo.toml @@ -25,6 +25,8 @@ vello_dev_macros = { workspace = true } bytemuck = { workspace = true } oxipng = { workspace = true, features = ["freestanding", "parallel"] } image = { workspace = true, features = ["png"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } skrifa = { workspace = true } smallvec = { workspace = true } diff --git a/sparse_strips/vello_sparse_tests/tests/util.rs b/sparse_strips/vello_sparse_tests/tests/util.rs index 33ec30086..f3171e967 100644 --- a/sparse_strips/vello_sparse_tests/tests/util.rs +++ b/sparse_strips/vello_sparse_tests/tests/util.rs @@ -21,6 +21,33 @@ use vello_cpu::{Level, RenderMode}; #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; +/// Aggregate diff report with statistics and individual pixel differences. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Debug, serde::Serialize)] +pub(crate) struct DiffReport { + /// Total number of pixels that differ. + pub pixel_count: usize, + /// Maximum absolute difference per channel [R, G, B, A]. + pub max_difference: [i16; 4], + /// Individual pixel differences. + pub pixels: Vec, +} + +/// Represents a single pixel difference between reference and actual images. +#[derive(Debug, serde::Serialize)] +pub(crate) struct PixelDiff { + /// The x coordinate of the differing pixel. + pub x: u32, + /// The y coordinate of the differing pixel. + pub y: u32, + /// The RGBA values from the reference image. + pub reference: [u8; 4], + /// The RGBA values from the actual image. + pub actual: [u8; 4], + /// Per-channel difference (actual - reference) as signed values. + pub difference: [i16; 4], +} + #[cfg(not(target_arch = "wasm32"))] static REFS_PATH: std::sync::LazyLock = std::sync::LazyLock::new(|| { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vello_sparse_tests/snapshots") @@ -321,9 +348,9 @@ pub(crate) fn check_ref( .into_rgba8(); let actual = load_from_memory(&encoded_image).unwrap().into_rgba8(); - let diff_image = get_diff(&ref_image, &actual, threshold, diff_pixels); + let diff_result = get_diff(&ref_image, &actual, threshold, diff_pixels); - if let Some(diff_image) = diff_image { + if let Some((diff_image, diff_data)) = diff_result { if should_replace() && is_reference { write_ref_image(); panic!("test was replaced"); @@ -338,6 +365,22 @@ pub(crate) fn check_ref( .save_with_format(&diff_path, image::ImageFormat::Png) .unwrap(); + // Save diff data as JSON + let json_path = DIFFS_PATH.join(format!("{specific_name}.json")); + let max_difference: [i16; 4] = diff_data.iter().fold([0; 4], |mut max, p| { + for (m, d) in max.iter_mut().zip(&p.difference) { + *m = (*m).max(d.abs()); + } + max + }); + let report = DiffReport { + pixel_count: diff_data.len(), + max_difference, + pixels: diff_data, + }; + let json_data = serde_json::to_string_pretty(&report).unwrap(); + std::fs::write(&json_path, json_data).unwrap(); + panic!("test didnt match reference image"); } } @@ -365,7 +408,7 @@ pub(crate) fn check_ref( let ref_image = load_from_memory(ref_data).unwrap().into_rgba8(); let diff_image = get_diff(&ref_image, &actual, threshold, diff_pixels); - if let Some(ref img) = diff_image { + if let Some((ref img, _)) = diff_image { append_diff_image_to_browser_document(specific_name, img); panic!("test didn't match reference image. Scroll to bottom of browser to view diff."); } @@ -445,11 +488,12 @@ fn get_diff( actual_image: &RgbaImage, threshold: u8, diff_pixels: u16, -) -> Option { +) -> Option<(RgbaImage, Vec)> { let width = max(expected_image.width(), actual_image.width()); let height = max(expected_image.height(), actual_image.height()); let mut diff_image = RgbaImage::new(width * 3, height); + let mut diff_data = Vec::new(); let mut pixel_diff = 0; @@ -465,6 +509,18 @@ fn get_diff( if is_pix_diff(expected, actual, threshold) { pixel_diff += 1; diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: expected.0, + actual: actual.0, + difference: [ + i16::from(actual.0[0]) - i16::from(expected.0[0]), + i16::from(actual.0[1]) - i16::from(expected.0[1]), + i16::from(actual.0[2]) - i16::from(expected.0[2]), + i16::from(actual.0[3]) - i16::from(expected.0[3]), + ], + }); } else { diff_image.put_pixel(x + width, y, Rgba([0, 0, 0, 255])); } @@ -473,23 +529,54 @@ fn get_diff( pixel_diff += 1; diff_image.put_pixel(x + 2 * width, y, *actual); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: [0, 0, 0, 0], + actual: actual.0, + difference: [ + i16::from(actual.0[0]), + i16::from(actual.0[1]), + i16::from(actual.0[2]), + i16::from(actual.0[3]), + ], + }); } (None, Some(expected)) => { pixel_diff += 1; diff_image.put_pixel(x, y, *expected); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: expected.0, + actual: [0, 0, 0, 0], + difference: [ + -i16::from(expected.0[0]), + -i16::from(expected.0[1]), + -i16::from(expected.0[2]), + -i16::from(expected.0[3]), + ], + }); } _ => { pixel_diff += 1; diff_image.put_pixel(x, y, Rgba([255, 0, 0, 255])); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: [0, 0, 0, 0], + actual: [0, 0, 0, 0], + difference: [0, 0, 0, 0], + }); } } } } if pixel_diff > diff_pixels { - Some(diff_image) + Some((diff_image, diff_data)) } else { None } From d9ec9a2e49674214c9e874bfdd13b9171118e383 Mon Sep 17 00:00:00 2001 From: Alex Gemberg Date: Mon, 22 Dec 2025 12:31:17 +1100 Subject: [PATCH 2/2] . --- Cargo.toml | 4 ++-- sparse_strips/vello_sparse_tests/tests/util.rs | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac201d0f4..adaf04439 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -154,5 +154,5 @@ console_log = "1.0" proc-macro2 = "1.0.95" syn = { version = "2.0.101", features = ["full", "extra-traits"] } quote = "1.0.40" -serde = { version = "1.0", default-features = false } -serde_json = "1.0" +serde = { version = "1.0.225", default-features = false } +serde_json = "1.0.145" diff --git a/sparse_strips/vello_sparse_tests/tests/util.rs b/sparse_strips/vello_sparse_tests/tests/util.rs index f3171e967..8b04527d9 100644 --- a/sparse_strips/vello_sparse_tests/tests/util.rs +++ b/sparse_strips/vello_sparse_tests/tests/util.rs @@ -381,7 +381,11 @@ pub(crate) fn check_ref( let json_data = serde_json::to_string_pretty(&report).unwrap(); std::fs::write(&json_path, json_data).unwrap(); - panic!("test didnt match reference image"); + panic!( + "test didn't match reference image\n diff image: {}\n diff report: {}", + diff_path.display(), + json_path.display() + ); } }