⚡️ Fix TDD SSE performance by splitting heavy comparison data

src/reporter/src/api/client.js

@@ -48,6 +48,15 @@ export const tdd = {
     return fetchJson('/api/report-data');
   },
 
+  /**
+   * Get full comparison details (lightweight + heavy fields)
+   * @param {string} id - Comparison ID, signature, or name
+   * @returns {Promise<Object>}
+   */
+  async getComparison(id) {
+    return fetchJson(`/api/comparison/${encodeURIComponent(id)}`);
+  },
+
   /**
    * Accept a single baseline
    * @param {string} id - Comparison ID

src/reporter/src/components/comparison/fullscreen-viewer.jsx

@@ -540,7 +540,8 @@ function FullscreenViewerInner({
             </button>
 
             {/* Regions toggle - only show if comparison has regions */}
-            {comparison?.confirmedRegions?.length > 0 && (
+            {(comparison?.confirmedRegions?.length > 0 ||
+              comparison?.hasConfirmedRegions) && (
               <button
                 type="button"
                 onClick={() => setShowRegions(!showRegions)}
@@ -825,7 +826,8 @@ function FullscreenViewerInner({
           </button>
 
           {/* Regions toggle - mobile */}
-          {comparison?.confirmedRegions?.length > 0 && (
+          {(comparison?.confirmedRegions?.length > 0 ||
+            comparison?.hasConfirmedRegions) && (
             <button
               type="button"
               onClick={() => setShowRegions(!showRegions)}

src/reporter/src/components/views/comparison-detail-view.jsx

@@ -2,6 +2,7 @@ import { useCallback, useMemo } from 'react';
 import { useLocation, useRoute } from 'wouter';
 import {
   useAcceptBaseline,
+  useComparison,
   useDeleteComparison,
   useRejectBaseline,
   useReportData,
@@ -27,10 +28,10 @@ export default function ComparisonDetailView() {
     [reportData?.comparisons]
   );
 
-  // Find the comparison by ID from route params
+  // Find the lightweight comparison by ID from route params
   // Uses stable IDs (id, signature, or name) - not array indices which change with filters
-  const comparison = useMemo(() => {
-    const targetId = params?.id ? decodeURIComponent(params.id) : null;
+  let lightComparison = useMemo(() => {
+    let targetId = params?.id ? decodeURIComponent(params.id) : null;
     if (!targetId || comparisons.length === 0) {
       return null;
     }
@@ -41,6 +42,24 @@ export default function ComparisonDetailView() {
     );
   }, [params, comparisons]);
 
+  // Fetch full comparison details on-demand (includes heavy fields like diffClusters)
+  let { data: fullComparison } = useComparison(lightComparison?.id);
+
+  // Merge lightweight SSE data with on-demand heavy fields
+  let comparison = useMemo(() => {
+    if (!lightComparison) return null;
+    if (!fullComparison) return lightComparison;
+    return {
+      ...lightComparison,
+      diffClusters: fullComparison.diffClusters,
+      confirmedRegions: fullComparison.confirmedRegions,
+      intensityStats: fullComparison.intensityStats,
+      boundingBox: fullComparison.boundingBox,
+      regionAnalysis: fullComparison.regionAnalysis,
+      hotspotAnalysis: fullComparison.hotspotAnalysis,
+    };
+  }, [lightComparison, fullComparison]);
+
   // Simple navigation - just change the URL using stable IDs
   const handleNavigate = useCallback(
     targetComparison => {

src/reporter/src/hooks/queries/use-tdd-queries.js

@@ -3,6 +3,16 @@ import { tdd } from '../../api/client.js';
 import { queryKeys } from '../../lib/query-keys.js';
 import { SSE_STATE, useReportDataSSE } from '../use-sse.js';
 
+export function useComparison(id, options = {}) {
+  return useQuery({
+    queryKey: queryKeys.comparison(id),
+    queryFn: () => tdd.getComparison(id),
+    enabled: !!id,
+    staleTime: 10_000,
+    ...options,
+  });
+}
+
 export function useReportData(options = {}) {
   // Use SSE for real-time updates
   let { state: sseState } = useReportDataSSE({

src/reporter/src/lib/query-keys.js

@@ -11,6 +11,7 @@ export const queryKeys = {
   // TDD (local)
   tdd: ['tdd'],
   reportData: () => [...queryKeys.tdd, 'report'],
+  comparison: id => [...queryKeys.tdd, 'comparison', id],
 
   // Cloud
   cloud: ['cloud'],

src/server/handlers/tdd-handler.js

@@ -198,15 +198,54 @@ export const createTddHandler = (
 
   const tddService = new TddService(config, workingDir, setBaseline);
   const reportPath = join(workingDir, '.vizzly', 'report-data.json');
+  const detailsPath = join(workingDir, '.vizzly', 'comparison-details.json');
+
+  /**
+   * Read heavy comparison details from comparison-details.json
+   * Returns a map of comparison ID -> heavy fields
+   */
+  const readComparisonDetails = () => {
+    try {
+      if (!existsSync(detailsPath)) return {};
+      return JSON.parse(readFileSync(detailsPath, 'utf8'));
+    } catch (error) {
+      output.debug('Failed to read comparison details:', error);
+      return {};
+    }
+  };
+
+  /**
+   * Persist heavy fields for a comparison to comparison-details.json
+   * This file is NOT watched by SSE, so writes here don't trigger broadcasts
+   * Skips writing if all heavy fields are empty (passed comparisons)
+   */
+  const updateComparisonDetails = (id, heavyFields) => {
+    let hasData = Object.values(heavyFields).some(
+      v => v != null && (!Array.isArray(v) || v.length > 0)
+    );
+    if (!hasData) return;
+
+    let details = readComparisonDetails();
+    details[id] = heavyFields;
+    writeFileSync(detailsPath, JSON.stringify(details));
+  };
+
+  /**
+   * Remove a comparison's heavy fields from comparison-details.json
+   */
+  const removeComparisonDetails = id => {
+    let details = readComparisonDetails();
+    delete details[id];
+    writeFileSync(detailsPath, JSON.stringify(details));
+  };
 
   const readReportData = () => {
     try {
       if (!existsSync(reportPath)) {
         return {
           timestamp: Date.now(),
-          comparisons: [], // Internal flat list for easy updates
-          groups: [], // Grouped structure for UI
-          summary: { total: 0, groups: 0, passed: 0, failed: 0, errors: 0 },
+          comparisons: [],
+          summary: { total: 0, passed: 0, failed: 0, errors: 0 },
         };
       }
       const data = readFileSync(reportPath, 'utf8');
@@ -216,8 +255,7 @@ export const createTddHandler = (
       return {
         timestamp: Date.now(),
         comparisons: [],
-        groups: [],
-        summary: { total: 0, groups: 0, passed: 0, failed: 0, errors: 0 },
+        summary: { total: 0, passed: 0, failed: 0, errors: 0 },
       };
     }
   };
@@ -254,14 +292,10 @@ export const createTddHandler = (
         });
       }
 
-      // Generate grouped structure from flat comparisons
-      reportData.groups = groupComparisons(reportData.comparisons);
-
-      // Update summary
+      // Update summary (groups computed client-side from comparisons)
       reportData.timestamp = Date.now();
       reportData.summary = {
         total: reportData.comparisons.length,
-        groups: reportData.groups.length,
         passed: reportData.comparisons.filter(
           c =>
             c.status === 'passed' ||
@@ -275,7 +309,7 @@ export const createTddHandler = (
         errors: reportData.comparisons.filter(c => c.status === 'error').length,
       };
 
-      writeFileSync(reportPath, JSON.stringify(reportData, null, 2));
+      writeFileSync(reportPath, JSON.stringify(reportData));
     } catch (error) {
       output.error('Failed to update comparison:', error);
     }
@@ -465,22 +499,46 @@ export const createTddHandler = (
     const vizzlyDir = join(workingDir, '.vizzly');
 
     // Record the comparison for the dashboard
-    // Spread the full comparison to include regionAnalysis, confirmedRegions, hotspotAnalysis, etc.
+    // Only include lightweight fields in report-data.json (broadcast via SSE)
     const newComparison = {
-      ...comparison,
-      originalName: name,
-      // Convert absolute file paths to web-accessible URLs
+      id: comparison.id,
+      name: comparison.name,
+      status: comparison.status,
+      signature: comparison.signature,
       baseline: convertPathToUrl(comparison.baseline, vizzlyDir),
       current: convertPathToUrl(comparison.current, vizzlyDir),
       diff: convertPathToUrl(comparison.diff, vizzlyDir),
-      // Use extracted properties with top-level viewport_width/browser
       properties: extractedProperties,
+      threshold: comparison.threshold,
+      minClusterSize: comparison.minClusterSize,
+      diffPercentage: comparison.diffPercentage,
+      diffCount: comparison.diffCount,
+      reason: comparison.reason,
+      totalPixels: comparison.totalPixels,
+      aaPixelsIgnored: comparison.aaPixelsIgnored,
+      aaPercentage: comparison.aaPercentage,
+      heightDiff: comparison.heightDiff,
+      error: comparison.error,
+      originalName: name,
       timestamp: Date.now(),
+      // Boolean hints so UI can show toggle buttons without fetching heavy data
+      hasDiffClusters: comparison.diffClusters?.length > 0,
+      hasConfirmedRegions: comparison.confirmedRegions?.length > 0,
     };
 
-    // Update comparison in report data file
+    // Update lightweight comparison in report-data.json (triggers SSE broadcast)
     updateComparison(newComparison);
 
+    // Persist heavy fields separately (NOT broadcast via SSE)
+    updateComparisonDetails(comparison.id, {
+      diffClusters: comparison.diffClusters,
+      intensityStats: comparison.intensityStats,
+      boundingBox: comparison.boundingBox,
+      regionAnalysis: comparison.regionAnalysis,
+      hotspotAnalysis: comparison.hotspotAnalysis,
+      confirmedRegions: comparison.confirmedRegions,
+    });
+
     // Log screenshot event for menubar
     // Normalize status to match HTTP response ('failed' -> 'diff')
     let logStatus = comparison.status === 'failed' ? 'diff' : comparison.status;
@@ -759,10 +817,14 @@ export const createTddHandler = (
       const freshReportData = {
         timestamp: Date.now(),
         comparisons: [],
-        groups: [],
-        summary: { total: 0, groups: 0, passed: 0, failed: 0, errors: 0 },
+        summary: { total: 0, passed: 0, failed: 0, errors: 0 },
       };
-      writeFileSync(reportPath, JSON.stringify(freshReportData, null, 2));
+      writeFileSync(reportPath, JSON.stringify(freshReportData));
+
+      // Clear comparison details
+      if (existsSync(detailsPath)) {
+        writeFileSync(detailsPath, JSON.stringify({}));
+      }
 
       output.info(
         `Baselines reset - ${deletedBaselines} baselines deleted, ${deletedCurrents} current screenshots deleted, ${deletedDiffs} diffs deleted`
@@ -843,17 +905,18 @@ export const createTddHandler = (
       output.warn(`Failed to update baseline metadata: ${error.message}`);
     }
 
+    // Remove heavy fields from comparison-details.json
+    removeComparisonDetails(comparisonId);
+
     // Remove comparison from report data
     reportData.comparisons = reportData.comparisons.filter(
       c => c.id !== comparisonId
     );
 
-    // Regenerate groups and summary
-    reportData.groups = groupComparisons(reportData.comparisons);
+    // Regenerate summary (groups computed client-side)
     reportData.timestamp = Date.now();
     reportData.summary = {
       total: reportData.comparisons.length,
-      groups: reportData.groups.length,
       passed: reportData.comparisons.filter(
         c =>
           c.status === 'passed' ||
@@ -866,7 +929,7 @@ export const createTddHandler = (
       errors: reportData.comparisons.filter(c => c.status === 'error').length,
     };
 
-    writeFileSync(reportPath, JSON.stringify(reportData, null, 2));
+    writeFileSync(reportPath, JSON.stringify(reportData));
 
     output.info(`Deleted comparison ${comparisonId} (${comparison.name})`);
     return { success: true, id: comparisonId };

src/server/routers/dashboard.js

@@ -6,7 +6,7 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import * as output from '../../utils/output.js';
-import { sendHtml, sendSuccess } from '../middleware/response.js';
+import { sendError, sendHtml, sendSuccess } from '../middleware/response.js';
 
 // SPA routes that should serve the dashboard HTML
 const SPA_ROUTES = ['/', '/stats', '/settings', '/projects', '/builds'];
@@ -70,6 +70,65 @@ export function createDashboardRouter(context) {
       }
     }
 
+    // API endpoint for fetching full comparison details (lightweight + heavy fields)
+    let comparisonMatch = pathname.match(/^\/api\/comparison\/(.+)$/);
+    if (comparisonMatch) {
+      let comparisonId = decodeURIComponent(comparisonMatch[1]);
+      if (!comparisonId) {
+        sendError(res, 400, 'Comparison ID is required');
+        return true;
+      }
+
+      let reportDataPath = join(workingDir, '.vizzly', 'report-data.json');
+      if (!existsSync(reportDataPath)) {
+        sendError(res, 404, 'No report data found');
+        return true;
+      }
+
+      try {
+        let reportData = JSON.parse(readFileSync(reportDataPath, 'utf8'));
+        let comparison = (reportData.comparisons || []).find(
+          c =>
+            c.id === comparisonId ||
+            c.signature === comparisonId ||
+            c.name === comparisonId
+        );
+
+        if (!comparison) {
+          sendError(res, 404, 'Comparison not found');
+          return true;
+        }
+
+        // Merge with heavy fields from comparison-details.json
+        let detailsPath = join(
+          workingDir,
+          '.vizzly',
+          'comparison-details.json'
+        );
+        if (existsSync(detailsPath)) {
+          try {
+            let details = JSON.parse(readFileSync(detailsPath, 'utf8'));
+            let heavy = details[comparison.id];
+            if (heavy) {
+              comparison = { ...comparison, ...heavy };
+            }
+          } catch (error) {
+            output.debug('Failed to read comparison details:', {
+              error: error.message,
+            });
+          }
+        }
+
+        sendSuccess(res, comparison);
+      } catch (error) {
+        output.debug('Error reading comparison data:', {
+          error: error.message,
+        });
+        sendError(res, 500, 'Failed to read comparison data');
+      }
+      return true;
+    }
+
     // Serve React SPA for dashboard routes
     if (SPA_ROUTES.includes(pathname) || pathname.startsWith('/comparison/')) {
       const reportDataPath = join(workingDir, '.vizzly', 'report-data.json');