feat(evaluator): aggregate agent-eval results natively and import Gym's own

packages/nemo_evaluator_sdk/examples/codex_docker/example.py

@@ -134,7 +134,7 @@ async def evaluate(
             output_dir=resolved_output_dir,
             parallelism=1,
             write_dashboard=write_dashboard,
-            benchmark={"name": "codex-docker-evidence-sanity"},
+            labels={"scenario": "codex-docker-evidence-sanity"},
         ),
     )
 

packages/nemo_evaluator_sdk/examples/gym/inspect_results.py

@@ -0,0 +1,202 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read and display the results of an agent-eval run.
+
+Companion to ``run_gym_eval.py``: that script *produces* a run bundle, this one *reads* it and shows
+how to get at each kind of result — headline aggregates, ``pass@k``, per-task outcomes, and the
+runner's own aggregations.
+
+The helpers below (:func:`aggregate`, :func:`per_task_outcomes`) are written to be lifted directly
+into your own code. Everything shown here also works on the in-memory ``AgentEvalResult`` returned by
+``AgentEvaluator().run(...)`` — reading from a bundle just makes the example runnable without a live
+run.
+
+Run from the repository root::
+
+    python -m packages.nemo_evaluator_sdk.examples.gym.inspect_results --bundle /tmp/gym-eval
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Sequence
+from pathlib import Path
+
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary
+from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore
+from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore
+
+#: Value at which an attempt counts as a pass, matching the SDK's pass@k definition (full credit).
+PASS_VALUE = 1.0
+
+#: Namespace the Gym runner's own aggregations are imported under, so they never collide with ours.
+RUNNER_PREFIX = "runner.gym."
+
+# --------------------------------------------------------------------------------------------------
+# Accessors — lift these into your own code.
+# --------------------------------------------------------------------------------------------------
+
+
+def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore:
+    """Look up one aggregate by name, e.g. ``"gym_reward.reward.pass@2"``.
+
+    Aggregates are a flat list, so this is a scan. Raises with the available names on a miss, which is
+    the failure you actually want when a metric or output was renamed.
+    """
+    for score in summary.scores.scores:
+        if score.name == name:
+            return score
+    available = ", ".join(sorted(score.name for score in summary.scores.scores))
+    raise KeyError(f"no aggregate named {name!r}; available: {available}")
+
+
+def per_task_outcomes(
+    scores: Sequence[AgentEvalTaskScore],
+    *,
+    metric_type: str,
+    output_name: str,
+) -> dict[str, list[float]]:
+    """Group per-trial score values by task: ``task_id -> [value per attempt]``.
+
+    A run with ``num_repeats=R`` produces R trials per task, and the scores are a flat
+    task x trial x metric list — so answering "which tasks failed?" means grouping them yourself.
+    """
+    by_task: dict[str, list[float]] = {}
+    for score in scores:
+        if score.metric_type != metric_type or score.status == AgentEvalScoreStatus.FAILED:
+            continue
+        for output in score.outputs:
+            if output.name == output_name and isinstance(output.value, int | float):
+                by_task.setdefault(score.task_id, []).append(float(output.value))
+    return by_task
+
+
+# --------------------------------------------------------------------------------------------------
+# Bundle loading (see the run.json manifest for the full artifact list).
+# --------------------------------------------------------------------------------------------------
+
+
+def load_bundle(bundle: Path) -> tuple[AgentEvalSummary, list[AgentEvalTaskScore]]:
+    """Hydrate the pieces of a persisted run bundle used below.
+
+    A runner's own numbers need no separate file: they are imported into ``summary.scores`` under
+    ``runner.<name>.``, so one load covers both.
+    """
+    summary = AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8")))
+    scores = [
+        AgentEvalTaskScore.model_validate(json.loads(line))
+        for line in (bundle / "scores.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    return summary, scores
+
+
+# --------------------------------------------------------------------------------------------------
+# Display
+# --------------------------------------------------------------------------------------------------
+
+
+def headline_value(score: AggregateScore) -> float | None:
+    """The one number for an aggregate: a scalar's ``value``, otherwise the mean of its distribution.
+
+    Scores named ``runner.<name>.*`` came from the runner rather than being computed here, and a
+    backend that reports a single figure (no underlying distribution) arrives as an
+    :class:`AggregateScalarScore` with no ``mean`` — so reading ``mean`` alone would show nothing.
+    """
+    return score.value if isinstance(score, AggregateScalarScore) else score.mean
+
+
+def show_aggregates(summary: AgentEvalSummary) -> None:
+    print("Aggregates ('runner.*' are the runner's own numbers, imported)")
+    print(f"  {'name':<40} {'value':>8} {'count':>6} {'nan':>5}")
+    for score in sorted(summary.scores.scores, key=lambda item: item.name):
+        value = headline_value(score)
+        shown = "—" if value is None else f"{value:.3f}"
+        # None means the producer didn't report a sample size; a real 0 means every sample was NaN.
+        count = "—" if score.count is None else str(score.count)
+        print(f"  {score.name:<40} {shown:>8} {count:>6} {score.nan_count:>5}")
+    print(f"\n  {summary.task_count} tasks · {summary.trial_count} trials · {summary.score_count} scores")
+
+
+def show_per_task(by_task: dict[str, list[float]]) -> None:
+    """Per-task outcomes: which tasks were solved, and how consistently.
+
+    An attempt passes on full credit (``>= PASS_VALUE``), matching how the SDK computes pass@k.
+    """
+    print("\nPer-task outcomes (attempt values; an attempt passes at full credit)")
+    solved = flaky = failed = 0
+    for task_id, values in sorted(by_task.items()):
+        passes = sum(1 for value in values if value >= PASS_VALUE)
+        if passes == len(values):
+            verdict, marker = "solved", "+"
+            solved += 1
+        elif passes:
+            verdict, marker = f"flaky ({passes}/{len(values)})", "~"
+            flaky += 1
+        else:
+            verdict, marker = "failed", "-"
+            failed += 1
+        attempts = ", ".join(f"{value:g}" for value in values)
+        print(f"  {marker} {task_id[:16]}…  [{attempts}]  {verdict}")
+    print(f"\n  {solved} solved · {flaky} flaky · {failed} failed")
+
+
+def show_runner_aggregations(summary: AgentEvalSummary) -> None:
+    """The runner's own numbers, plus a cross-check against the SDK's native aggregates.
+
+    Imported figures sit in the same ``summary.scores`` list as everything else, distinguished only by
+    the ``runner.`` prefix — so they are read exactly like the natively-computed ones. Units and names
+    stay the runner's own: Gym reports accuracy on a 0-100 scale where the SDK uses 0-1.
+    """
+    imported = [score for score in summary.scores.scores if score.name.startswith(RUNNER_PREFIX)]
+    if not imported:
+        print("\nNo runner-provided aggregations (this runner doesn't supply any).")
+        return
+
+    print("\nRunner-provided aggregations (imported into summary.scores)")
+    for score in sorted(imported, key=lambda item: item.name):
+        value = headline_value(score)
+        shown = "—" if value is None else f"{value:g}"
+        print(f"      {score.name[len(RUNNER_PREFIX) :]:<34} {shown}")
+
+    # Cross-check: the SDK computes pass@k natively from the trials; Gym computes its own. They should
+    # agree once you normalise the scale.
+    try:
+        native = aggregate(summary, "gym_reward.reward.pass@1").mean
+        reported = headline_value(aggregate(summary, f"{RUNNER_PREFIX}pass@1/accuracy"))
+    except KeyError:
+        return
+    if native is not None and reported is not None:
+        agreement = "agree" if abs(native - reported / 100) < 1e-9 else "DIFFER"
+        print(f"\n  cross-check pass@1: native={native:.3f} · runner={reported / 100:.3f} (0-100 scale) -> {agreement}")
+
+
+def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--bundle", type=Path, default=Path("/tmp/gym-eval"), help="Run bundle directory to read.")
+    parser.add_argument("--metric-type", default="gym_reward", help="Metric type to break down per task.")
+    parser.add_argument("--output-name", default="reward", help="Metric output to break down per task.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(argv)
+    if not (args.bundle / "summary.json").exists():
+        raise SystemExit(f"{args.bundle} is not a run bundle (no summary.json). Run run_gym_eval.py first.")
+
+    summary, scores = load_bundle(args.bundle)
+
+    show_aggregates(summary)
+    by_task = per_task_outcomes(scores, metric_type=args.metric_type, output_name=args.output_name)
+    if by_task:
+        show_per_task(by_task)
+    show_runner_aggregations(summary)
+
+    print(f"\nFull report: {args.bundle / 'report.html'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

packages/nemo_evaluator_sdk/examples/profbench/README.md

@@ -511,6 +511,6 @@ What happens in the full live-candidate path:
 4. `load_profbench()` loads tasks with `include_cached_fulfilments=False`, so cached labels are removed and the metric must call the judge.
 5. `AgentEvaluator.run(tasks=..., target=evaluated_model, ...)` generates fresh candidate trials: for each task it calls `_generate_sample()` against the evaluated model and converts the returned sample into an `AgentEvalTrial`.
 6. The evaluator then scores those generated trials with `ProfBenchRubricMetric`. For each rubric criterion it calls `ProfBenchModelJudge`, which calls `_generate_sample()` against the judge model, parses the judge output into a yes/no decision, writes `evidence/judge-*.json`, and returns `MetricResult` outputs.
-7. `AgentEvaluator` builds the summary, persists the run bundle (`benchmark.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`.
+7. `AgentEvaluator` builds the summary, persists the run bundle (`metadata.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`.
 
 The evaluated model produces the candidate answer and the judge model evaluates each rubric criterion. They can point to the same model configuration, but the code treats them as separate roles.

packages/nemo_evaluator_sdk/examples/profbench/runner.py

@@ -100,7 +100,7 @@ async def run_profbench_mode(
     target: AgentEvalTarget | None = None
     trials: list[AgentEvalTrial] | None = None
     params: RunConfigOnlineModel | None = None
-    benchmark_meta = dict(benchmark.metadata)
+    benchmark_labels = {key: str(value) for key, value in benchmark.metadata.items()}
     if mode is ProfBenchMode.LIVE_CANDIDATE:
         target, params, score_source, effective_codex_runtime = _live_candidate_target(
             agent=agent,
@@ -110,11 +110,11 @@ async def run_profbench_mode(
         )
         if effective_codex_runtime is not None:
             print(f"Codex runtime: {effective_codex_runtime}")
-        benchmark_meta["score_source"] = score_source
+        benchmark_labels["score_source"] = score_source
     else:
         trials = benchmark.trials
         if mode is ProfBenchMode.LIVE_JUDGE:
-            benchmark_meta["score_source"] = "live_judge"
+            benchmark_labels["score_source"] = "live_judge"
 
     result = await AgentEvaluator().run(
         tasks=benchmark.tasks,
@@ -124,7 +124,7 @@ async def run_profbench_mode(
             output_dir=output_dir,
             run_id=f"{run_instance_id}-{mode.value}",
             params=params,
-            benchmark=benchmark_meta,
+            labels=benchmark_labels,
             write_dashboard=False,
         ),
     )

packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py

@@ -143,7 +143,7 @@ def summarize_run(
     total = len(task_ids)
     return {
         "run_id": result.run_id,
-        "benchmark": result.benchmark,
+        "labels": result.metadata.labels,
         "total_tasks": total,
         "passed_tasks": passed,
         "pass_rate": (passed / total) if total else 0.0,

packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py

@@ -58,7 +58,7 @@ async def run_tasks(
         tasks: Sequence[AgentEvalTask],
         *,
         target: AgentEvalTarget,
-        benchmark: dict[str, object] | None = None,
+        labels: dict[str, str] | None = None,
         output_dir: Path | None = None,
         run_id: str | None = None,
         prepare_task: Callable[[AgentEvalTask], None] | None = None,
@@ -72,7 +72,7 @@ async def run_tasks(
         result = await AgentEvaluator().run(
             tasks=prepared,
             target=target,
-            config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
+            config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels),
         )
         self._maybe_write_gate(result)
         return result
@@ -82,7 +82,7 @@ async def score_trials(
         tasks: Sequence[AgentEvalTask],
         *,
         trials: Sequence[AgentEvalTrial],
-        benchmark: dict[str, object] | None = None,
+        labels: dict[str, str] | None = None,
         output_dir: Path | None = None,
         run_id: str | None = None,
     ) -> AgentEvalResult:
@@ -91,7 +91,7 @@ async def score_trials(
         result = await AgentEvaluator().run(
             tasks=prepared,
             trials=list(trials),
-            config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
+            config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels),
         )
         self._maybe_write_gate(result)
         return result
@@ -101,14 +101,14 @@ def _run_config(
         *,
         output_dir: Path | None,
         run_id: str | None,
-        benchmark: dict[str, object] | None,
+        labels: dict[str, str] | None,
     ) -> AgentEvalRunConfig:
         return AgentEvalRunConfig(
             output_dir=output_dir,
             run_id=run_id,
             parallelism=self.config.parallelism,
             write_dashboard=self.config.write_dashboard,
-            benchmark=dict(benchmark or {}),
+            labels=dict(labels or {}),
         )
 
     def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask:

packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py

@@ -77,7 +77,7 @@ async def run_online(task_names: list[str], *, output_dir: Path, min_pass_rate:
     return await _pipeline(min_pass_rate).run_tasks(
         tasks,
         target=runtime,
-        benchmark={"benchmark": "run-agent-eval", "mode": "online"},
+        labels={"example": "run-agent-eval", "mode": "online"},
         output_dir=output_dir,
     )
 
@@ -132,7 +132,7 @@ async def run_agentic_task(
     return await _pipeline(min_pass_rate, extra_metrics=extra_metrics).run_tasks(
         [task],
         target=runtime,
-        benchmark={"benchmark": "agentic-use", "task": task_name, "backend": backend},
+        labels={"example": "agentic-use", "task": task_name, "backend": backend},
         output_dir=output_dir,
         prepare_task=lambda t: ensure_task_image(t, skip_build=skip_build),
     )
@@ -145,7 +145,7 @@ async def rescore(rescore_dirs: list[Path], *, output_dir: Path, min_pass_rate:
     return await _pipeline(min_pass_rate).score_trials(
         tasks,
         trials=trials,
-        benchmark={"benchmark": "run-agent-eval", "mode": "offline"},
+        labels={"example": "run-agent-eval", "mode": "offline"},
         output_dir=output_dir,
     )
 

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/dashboard.py

@@ -12,6 +12,7 @@
 
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult
 from nemo_evaluator_sdk.agent_eval.scores import AgentEvalTaskScore
+from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore
 from pydantic import BaseModel
 
 
@@ -83,18 +84,41 @@ def _metric_rollups(result: AgentEvalResult) -> str:
         rows.append(
             "<tr>"
             f"<td><code>{_e(score.name)}</code></td>"
-            f"<td>{_format_score(score.mean)}</td>"
-            f"<td>{_e(score.count)}</td>"
+            f"<td>{_format_score(_headline_value(score))}</td>"
+            f"<td>{_format_score(_median(score))}</td>"
+            f"<td>{_format_score(score.sample_std_dev)}</td>"
+            f"<td>{_count(score.count)}</td>"
             f"<td>{_e(score.nan_count)}</td>"
             "</tr>"
         )
     return (
-        "<table><thead><tr><th>Name</th><th>Mean</th><th>Count</th><th>NaN</th></tr></thead><tbody>"
-        + "".join(rows)
-        + "</tbody></table>"
+        "<table><thead><tr><th>Name</th><th>Value</th><th>Median</th><th>Std dev</th>"
+        "<th>Count</th><th>NaN</th></tr></thead><tbody>" + "".join(rows) + "</tbody></table>"
     )
 
 
+def _headline_value(score: AggregateScore) -> float | None:
+    """The one number to show: a scalar's ``value``, otherwise the mean of the distribution.
+
+    A scalar score has no mean — rendering the column straight off ``score.mean`` would leave every
+    runner-imported figure blank in the table where it is the only thing worth reading.
+    """
+    return score.value if isinstance(score, AggregateScalarScore) else score.mean
+
+
+def _median(score: AggregateScore) -> float | None:
+    percentiles = getattr(score, "percentiles", None)
+    return percentiles.p50 if percentiles is not None else None
+
+
+def _count(count: int | None) -> str:
+    """Sample size, or an em dash when the producer didn't report one (imported aggregates).
+
+    Tests for None specifically: a genuine 0 means every sample was NaN, which is worth seeing.
+    """
+    return "&mdash;" if count is None else _e(count)
+
+
 def _score_table(scores: list[AgentEvalTaskScore]) -> str:
     if not scores:
         return '<p class="muted">No metric scores.</p>'