report: per-submission breakdown of measurements and checks (#231)

scripts/cape-subcommands/submission/benchmark.html.tmpl

@@ -235,7 +235,6 @@
     <script>
     (function () {
         const BENCHMARK = "{{.benchmark}}";
-        const REPO_MAIN = "https://github.com/IntersectMBO/UPLC-CAPE/blob/main";
 
         // `exceedsBudget` says whether a submission blew the Cardano protocol limit for this metric.
         // Exceeding submissions get a full-width hatched bar; non-exceeding bars are normalised
@@ -310,9 +309,11 @@
             return parts.join("");
         }
 
+        // Link to the per-submission evaluation breakdown page emitted alongside
+        // this scenario page at benchmarks/<scenario>/<submission_dir>.html.
         function submissionLink(sub) {
             if (!sub.submission_dir) return null;
-            return `${REPO_MAIN}/submissions/${BENCHMARK}/${sub.submission_dir}`;
+            return `${encodeURIComponent(BENCHMARK)}/${encodeURIComponent(sub.submission_dir)}.html`;
         }
 
         function sortForMetric(submissions, metric) {
@@ -359,7 +360,7 @@
                 const labelHtml = submissionLabelHtml(sub);
                 const link = submissionLink(sub);
                 const labelCell = link
-                    ? `<a href="${link}" target="_blank" rel="noopener">${labelHtml}</a>`
+                    ? `<a href="${link}" title="Per-evaluation breakdown">${labelHtml}</a>`
                     : labelHtml;
                 const valueEsc = escapeHtml(formatted);
                 const isEmpty = value === 0;
@@ -448,8 +449,8 @@
                 note.textContent =
                     `Aggregates cover expected on-chain evaluations only; ${excluded} ` +
                     `evaluations (failure-path checks and/or pending measurements) are ` +
-                    `measured per submission but excluded — see each submission's ` +
-                    `metrics.json for their cost.`;
+                    `measured per submission but excluded — open a submission's ` +
+                    `breakdown (click its label) for the per-evaluation cost.`;
             }
         }
 

scripts/cape-subcommands/submission/report.help.tmpl

@@ -14,6 +14,9 @@ Options:
 The report includes:
 - Main index page (report/index.html) listing all benchmarks
 - Individual benchmark report pages (report/benchmarks/<benchmark>.html)
+- Per-submission evaluation breakdown pages
+  (report/benchmarks/<benchmark>/<submission_dir>.html), linked from each
+  submission row on the benchmark page
 - 4 PNG chart images per benchmark comparing submissions by:
   - CPU Units: Execution cost in CPU units
   - Memory Units: Memory consumption
@@ -25,6 +28,7 @@ Each submission is labeled as: {language}_{version}_{user}
 Output:
   report/index.html                           # Main index with all benchmarks
   report/benchmarks/<benchmark>.html          # Individual benchmark reports
+  report/benchmarks/<benchmark>/<dir>.html    # Per-submission evaluation breakdowns
   report/benchmarks/images/<benchmark>_*.png  # Chart image files
 
 Examples:

scripts/cape-subcommands/submission/report.sh

@@ -139,26 +139,35 @@ fi
 # it, so we fill it eagerly via `aggregate_csv_cache` before entering any loop.
 declare -A AGGREGATE_CSV=()
 
-# Populate AGGREGATE_CSV[<target_filter>] if not already set. Call from the outer
+# Map a target filter to its cache key. The unfiltered case uses an empty
+# filter string, but bash 5.3 rejects an empty associative-array subscript
+# ("bad array subscript"), so fold it to a non-empty sentinel that can't
+# collide with a real target name.
+csv_cache_key() { printf '%s' "${1:-__unfiltered__}"; }
+
+# Populate AGGREGATE_CSV[<key>] if not already set. Call from the outer
 # shell before using fetch_benchmark_csv in a $(...) context.
 aggregate_csv_cache() {
   local target_filter="${1:-}"
-  if [ -n "${AGGREGATE_CSV[$target_filter]+set}" ]; then
+  local key
+  key=$(csv_cache_key "$target_filter")
+  if [ -n "${AGGREGATE_CSV[$key]+set}" ]; then
     return 0
   fi
   local aggregate_args=()
   if [ -n "$target_filter" ]; then
     aggregate_args+=("--target=$target_filter")
   fi
-  AGGREGATE_CSV[$target_filter]=$($CAPE_CMD submission aggregate "${aggregate_args[@]}")
+  AGGREGATE_CSV[$key]=$($CAPE_CMD submission aggregate "${aggregate_args[@]}")
 }
 
 # Fetch CSV rows for one benchmark, dropping template placeholders and rows with empty core metrics.
 fetch_benchmark_csv() {
   local benchmark="$1"
   local target_filter="${2:-}"
-  local csv_data
-  csv_data=$(printf '%s' "${AGGREGATE_CSV[$target_filter]:-}" | grep "^$benchmark," || true)
+  local key csv_data
+  key=$(csv_cache_key "$target_filter")
+  csv_data=$(printf '%s' "${AGGREGATE_CSV[$key]:-}" | grep "^$benchmark," || true)
   if [ -z "$csv_data" ]; then
     return 0
   fi
@@ -324,6 +333,117 @@ EOF
   fi
 }
 
+# Render per-submission evaluation breakdown pages under
+# report/benchmarks/<benchmark>/<submission_dir>.html. Each page lists every
+# evaluation from that submission's metrics.json (measurements = included in
+# aggregates, checks = excluded). The breakdown is read directly from
+# metrics.json rather than the positional aggregate CSV, which can't carry a
+# variable-length evaluation list.
+generate_submission_detail_pages() {
+  local benchmark="$1"
+  local output_dir="$2"
+  local target_filter="${3:-}"
+
+  local csv_data
+  csv_data=$(fetch_benchmark_csv "$benchmark" "$target_filter")
+  [ -n "$csv_data" ] || return 0
+
+  local detail_dir="$output_dir/benchmarks/$benchmark"
+  local abs_template_path="$SCRIPT_DIR/submission-detail.html.tmpl"
+
+  if [[ $DRY_RUN -eq 1 ]]; then
+    log_info "[dry-run] Would render submission detail pages under $detail_dir/"
+    return 0
+  fi
+
+  local ts
+  ts=$(date '+%Y-%m-%d %H:%M:%S %Z')
+  mkdir -p "$detail_dir"
+
+  while IFS= read -r line; do
+    [ -n "$line" ] || continue
+    local language version user variant submission_dir
+    language=$(csv_field "$line" "language")
+    version=$(csv_field "$line" "version")
+    user=$(csv_field "$line" "user")
+    variant=$(csv_field "$line" "variant")
+    submission_dir=$(csv_field "$line" "submission_dir")
+
+    [ -n "$submission_dir" ] || continue
+    local metrics_path="$PROJECT_ROOT/submissions/$benchmark/$submission_dir/metrics.json"
+    if [ ! -f "$metrics_path" ]; then
+      log_warn "metrics.json not found for $benchmark/$submission_dir, skipping detail page"
+      continue
+    fi
+
+    local label="$language $version"
+    if [ -n "$variant" ] && [ "$variant" != "default" ]; then
+      label="$label · $variant"
+    fi
+    label="$label · $user"
+
+    local source_url="https://github.com/IntersectMBO/UPLC-CAPE/blob/main/submissions/$benchmark/$submission_dir"
+
+    local temp_json
+    temp_json=$(mktemp_json)
+    jq \
+      --arg benchmark "$benchmark" \
+      --arg submission_dir "$submission_dir" \
+      --arg label "$label" \
+      --arg compiler "$language" \
+      --arg version "$version" \
+      --arg variant "$variant" \
+      --arg user "$user" \
+      --arg source_url "$source_url" \
+      --arg timestamp "$ts" \
+      '
+      def commafy:
+        (tostring | explode | reverse) as $rev
+        | [range(0; ($rev | length)) as $i
+           | (($rev[$i:$i+1]) | implode)
+             + (if ($i > 0 and ($i % 3 == 0)) then "," else "" end)]
+        | reverse | join("");
+      # gomplate renders *.html.tmpl with Go text/template (no auto-escaping),
+      # so every interpolated string must be HTML-escaped here. Free-form
+      # fields like an evaluation description can legitimately contain < > &.
+      def esc:
+        gsub("&"; "&amp;") | gsub("<"; "&lt;") | gsub(">"; "&gt;")
+        | gsub("\""; "&quot;") | gsub("'"'"'"; "&#39;");
+      def row: {
+        name: (.name | esc),
+        description: (.description | esc),
+        execution_result: (.execution_result | esc),
+        cpu_units, memory_units,
+        cpu_fmt: (.cpu_units | commafy),
+        mem_fmt: (.memory_units | commafy)
+      };
+      (.evaluations // []) as $evals
+      | ($evals | map(select(.included_in_aggregates))       | map(row)) as $measurements
+      | ($evals | map(select(.included_in_aggregates | not)) | map(row)) as $checks
+      | {
+          benchmark: ($benchmark | esc),
+          submission_dir: ($submission_dir | esc),
+          label: ($label | esc),
+          compiler: ($compiler | esc),
+          version: ($version | esc),
+          variant: ($variant | esc),
+          user: ($user | esc),
+          source_url: ($source_url | esc),
+          timestamp: ($timestamp | esc),
+          measurements: $measurements,
+          checks: $checks,
+          measurements_count: ($measurements | length),
+          checks_count: ($checks | length)
+        }
+      ' "$metrics_path" > "$temp_json"
+
+    if ! gomplate -f "$abs_template_path" -c .="$temp_json" > "$detail_dir/${submission_dir}.html"; then
+      echo "ERROR: Submission detail template rendering failed for $benchmark/$submission_dir" >&2
+      exit 1
+    fi
+  done <<< "$csv_data"
+}
+
 # Build filtered benchmark stats from CSV data (consumed by index.html.tmpl).
 build_filtered_stats() {
   local target_filter="${1:-}"
@@ -591,6 +711,10 @@ generate_index_report() {
 EOF
 
   local first=true
+  # `benchmark` is localized so this loop doesn't clobber the caller's
+  # `benchmark` variable (the single-benchmark entry point reads it after
+  # this returns, for its success messages).
+  local benchmark
   while read -r benchmark; do
     if [ -n "$benchmark" ]; then
       if [ "$first" = "false" ]; then
@@ -860,6 +984,8 @@ if [ "$1" = "--all" ]; then
         generate_individual_benchmark_report "$benchmark" "$report_dir"
       fi
 
+      generate_submission_detail_pages "$benchmark" "$report_dir" "current"
+
       if [ -n "$benchmarks_with_submissions" ]; then
         benchmarks_with_submissions="${benchmarks_with_submissions}\n"
       fi
@@ -877,6 +1003,7 @@ if [ "$1" = "--all" ]; then
     echo "   📄 $report_dir/index.html (main index)"
     echo "   📊 Individual benchmark reports in $report_dir/benchmarks/"
     echo "   📦 Benchmark data JSON in $report_dir/benchmarks/<scenario>.json"
+    echo "   🔍 Per-submission breakdowns in $report_dir/benchmarks/<scenario>/<submission_dir>.html"
   else
     echo "Error: No submissions found for report generation" >&2
     exit 1
@@ -960,6 +1087,7 @@ else
     fi
 
     generate_individual_benchmark_report "$benchmark" "$report_dir"
+    generate_submission_detail_pages "$benchmark" "$report_dir"
   else
     echo "No submissions found for benchmark: $benchmark. Creating placeholder page."
     generate_no_submissions_report "$benchmark" "$report_dir"
@@ -971,6 +1099,7 @@ else
   echo "   📄 $report_dir/index.html (main index)"
   echo "   📊 Individual benchmark report: $report_dir/benchmarks/${benchmark}.html"
   echo "   📦 Benchmark data JSON: $report_dir/benchmarks/${benchmark}.json"
+  echo "   🔍 Per-submission breakdowns: $report_dir/benchmarks/${benchmark}/<submission_dir>.html"
   echo ""
   echo "Open the main report with: xdg-open $report_dir/index.html 2>/dev/null || echo \"Open: $report_dir/index.html\""
 fi