diff --git a/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py new file mode 100644 index 0000000000..e9cb525762 --- /dev/null +++ b/packages/nemo_evaluator_sdk/examples/gym/inspect_results.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read and display the results of an agent-eval run. + +Companion to ``run_gym_eval.py``: that script *produces* a run bundle, this one *reads* it and shows +how to get at each kind of result — headline aggregates, ``pass@k``, per-task outcomes, and the +runner's own aggregations. + +The helpers below (:func:`aggregate`, :func:`per_task_outcomes`) are written to be lifted directly +into your own code. Everything shown here also works on the in-memory ``AgentEvalResult`` returned by +``AgentEvaluator().run(...)`` — reading from a bundle just makes the example runnable without a live +run. + +Run from the repository root:: + + uv run python -m packages.nemo_evaluator_sdk.examples.gym.inspect_results --bundle /tmp/gym-eval +""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence +from pathlib import Path + +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore +from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore + +#: Value at which an attempt counts as a pass, matching the SDK's pass@k definition (full credit). +PASS_VALUE = 1.0 + +#: Namespace the Gym runner's own aggregations are imported under, so they never collide with ours. +RUNNER_PREFIX = "runner.gym." + +# -------------------------------------------------------------------------------------------------- +# Accessors — lift these into your own code. +# -------------------------------------------------------------------------------------------------- + + +def aggregate(summary: AgentEvalSummary, name: str) -> AggregateScore: + """Look up one aggregate by name, e.g. ``"gym_reward.reward.pass@2"``. + + Aggregates are a flat list, so this is a scan. Raises with the available names on a miss, which is + the failure you actually want when a metric or output was renamed. + """ + for score in summary.scores.scores: + if score.name == name: + return score + available = ", ".join(sorted(score.name for score in summary.scores.scores)) + raise KeyError(f"no aggregate named {name!r}; available: {available}") + + +def per_task_outcomes( + scores: Sequence[AgentEvalTaskScore], + *, + metric_type: str, + output_name: str, +) -> dict[str, list[float]]: + """Group per-trial score values by task: ``task_id -> [value per attempt]``. + + A run with ``num_repeats=R`` produces R trials per task, and the scores are a flat + task x trial x metric list — so answering "which tasks failed?" means grouping them yourself. + """ + by_task: dict[str, list[float]] = {} + for score in scores: + if score.metric_type != metric_type or score.status == AgentEvalScoreStatus.FAILED: + continue + for output in score.outputs: + if output.name == output_name and isinstance(output.value, int | float): + by_task.setdefault(score.task_id, []).append(float(output.value)) + return by_task + + +# -------------------------------------------------------------------------------------------------- +# Bundle loading (see the run.json manifest for the full artifact list). +# -------------------------------------------------------------------------------------------------- + + +def load_bundle(bundle: Path) -> tuple[AgentEvalSummary, list[AgentEvalTaskScore]]: + """Hydrate the pieces of a persisted run bundle used below. + + A runner's own numbers need no separate file: they are imported into ``summary.scores`` under + ``runner..``, so one load covers both. + """ + summary = AgentEvalSummary.model_validate(json.loads((bundle / "summary.json").read_text(encoding="utf-8"))) + scores = [ + AgentEvalTaskScore.model_validate(json.loads(line)) + for line in (bundle / "scores.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + return summary, scores + + +# -------------------------------------------------------------------------------------------------- +# Display +# -------------------------------------------------------------------------------------------------- + + +def headline_value(score: AggregateScore) -> float | None: + """The one number for an aggregate: a scalar's ``value``, otherwise the mean of its distribution. + + Scores named ``runner..*`` came from the runner rather than being computed here, and a + backend that reports a single figure (no underlying distribution) arrives as an + :class:`AggregateScalarScore` with no ``mean`` — so reading ``mean`` alone would show nothing. + """ + return score.value if isinstance(score, AggregateScalarScore) else score.mean + + +def show_aggregates(summary: AgentEvalSummary) -> None: + print("Aggregates ('runner.*' are the runner's own numbers, imported)") + print(f" {'name':<40} {'value':>8} {'count':>6} {'nan':>5}") + for score in sorted(summary.scores.scores, key=lambda item: item.name): + value = headline_value(score) + shown = "—" if value is None else f"{value:.3f}" + # None means the producer didn't report a sample size; a real 0 means every sample was NaN. + count = "—" if score.count is None else str(score.count) + print(f" {score.name:<40} {shown:>8} {count:>6} {score.nan_count:>5}") + print(f"\n {summary.task_count} tasks · {summary.trial_count} trials · {summary.score_count} scores") + + +def show_per_task(by_task: dict[str, list[float]]) -> None: + """Per-task outcomes: which tasks were solved, and how consistently. + + An attempt passes on full credit (``>= PASS_VALUE``), matching how the SDK computes pass@k. + """ + print("\nPer-task outcomes (attempt values; an attempt passes at full credit)") + solved = flaky = failed = 0 + for task_id, values in sorted(by_task.items()): + passes = sum(1 for value in values if value >= PASS_VALUE) + if passes == len(values): + verdict, marker = "solved", "+" + solved += 1 + elif passes: + verdict, marker = f"flaky ({passes}/{len(values)})", "~" + flaky += 1 + else: + verdict, marker = "failed", "-" + failed += 1 + attempts = ", ".join(f"{value:g}" for value in values) + print(f" {marker} {task_id[:16]}… [{attempts}] {verdict}") + print(f"\n {solved} solved · {flaky} flaky · {failed} failed") + + +def show_runner_aggregations(summary: AgentEvalSummary) -> None: + """The runner's own numbers, plus a cross-check against the SDK's native aggregates. + + Imported figures sit in the same ``summary.scores`` list as everything else, distinguished only by + the ``runner.`` prefix — so they are read exactly like the natively-computed ones. Units and names + stay the runner's own: Gym reports accuracy on a 0-100 scale where the SDK uses 0-1. + """ + imported = [score for score in summary.scores.scores if score.name.startswith(RUNNER_PREFIX)] + if not imported: + print("\nNo runner-provided aggregations (this runner doesn't supply any).") + return + + print("\nRunner-provided aggregations (imported into summary.scores)") + for score in sorted(imported, key=lambda item: item.name): + value = headline_value(score) + shown = "—" if value is None else f"{value:g}" + print(f" {score.name[len(RUNNER_PREFIX) :]:<34} {shown}") + + # Cross-check: the SDK computes pass@k natively from the trials; Gym computes its own. They should + # agree once you normalise the scale. + try: + native = aggregate(summary, "gym_reward.reward.pass@1").mean + reported = headline_value(aggregate(summary, f"{RUNNER_PREFIX}pass@1/accuracy")) + except KeyError: + return + if native is not None and reported is not None: + agreement = "agree" if abs(native - reported / 100) < 1e-9 else "DIFFER" + print(f"\n cross-check pass@1: native={native:.3f} · runner={reported / 100:.3f} (0-100 scale) -> {agreement}") + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--bundle", type=Path, default=Path("/tmp/gym-eval"), help="Run bundle directory to read.") + parser.add_argument("--metric-type", default="gym_reward", help="Metric type to break down per task.") + parser.add_argument("--output-name", default="reward", help="Metric output to break down per task.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + if not (args.bundle / "summary.json").exists(): + raise SystemExit(f"{args.bundle} is not a run bundle (no summary.json). Run run_gym_eval.py first.") + + summary, scores = load_bundle(args.bundle) + + show_aggregates(summary) + by_task = per_task_outcomes(scores, metric_type=args.metric_type, output_name=args.output_name) + if by_task: + show_per_task(by_task) + show_runner_aggregations(summary) + + print(f"\nFull report: {args.bundle / 'report.html'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/dashboard.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/dashboard.py index 571957c760..376509aaf7 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/dashboard.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/dashboard.py @@ -12,6 +12,7 @@ from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult from nemo_evaluator_sdk.agent_eval.scores import AgentEvalTaskScore +from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore from pydantic import BaseModel @@ -83,18 +84,41 @@ def _metric_rollups(result: AgentEvalResult) -> str: rows.append( "" f"{_e(score.name)}" - f"{_format_score(score.mean)}" - f"{_e(score.count)}" + f"{_format_score(_headline_value(score))}" + f"{_format_score(_median(score))}" + f"{_format_score(score.sample_std_dev)}" + f"{_count(score.count)}" f"{_e(score.nan_count)}" "" ) return ( - "" - + "".join(rows) - + "
NameMeanCountNaN
" + "" + "" + "".join(rows) + "
NameValueMedianStd devCountNaN
" ) +def _headline_value(score: AggregateScore) -> float | None: + """The one number to show: a scalar's ``value``, otherwise the mean of the distribution. + + A scalar score has no mean — rendering the column straight off ``score.mean`` would leave every + runner-imported figure blank in the table where it is the only thing worth reading. + """ + return score.value if isinstance(score, AggregateScalarScore) else score.mean + + +def _median(score: AggregateScore) -> float | None: + percentiles = getattr(score, "percentiles", None) + return percentiles.p50 if percentiles is not None else None + + +def _count(count: int | None) -> str: + """Sample size, or an em dash when the producer didn't report one (imported aggregates). + + Tests for None specifically: a genuine 0 means every sample was NaN, which is worth seeing. + """ + return "—" if count is None else _e(count) + + def _score_table(scores: list[AgentEvalTaskScore]) -> str: if not scores: return '

No metric scores.

' diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index c42b240bfe..69dddf5c9c 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -37,6 +37,7 @@ AgentEvalTrialStatus, AgentOutput, AgentTaskRunner, + RunAggregationsProvider, RunnerInfo, ) from nemo_evaluator_sdk.agent_inference import ( @@ -60,6 +61,7 @@ RunConfigOnline, RunConfigOnlineModel, ) +from nemo_evaluator_sdk.values.results import AggregateScore from nemo_evaluator_sdk.values.evidence import ( EVIDENCE_FORMAT_JSON, EVIDENCE_TRACE, @@ -176,6 +178,7 @@ async def run( config=runtime_config, run_id=run_id, ) + runner_scores = _collect_runner_aggregate_scores(target) if target is not None else [] finished_at = datetime.now(UTC) metadata = RunMetadata( labels=dict(runtime_config.labels), @@ -190,7 +193,7 @@ async def run( tasks=task_list, trials=trial_list, scores=scores, - summary=AgentEvalSummary.from_scores(scores, tasks=task_list), + summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores), metadata=metadata, ) @@ -732,6 +735,17 @@ def _describe_target( return target.runner_info() +def _collect_runner_aggregate_scores(target: object) -> list[AggregateScore]: + """The typed subset of a runner's own aggregations, for merging into ``summary.scores``. + + A runner that maps its numbers onto aggregate scores namespaces them under ``runner..``, so + they sit alongside the SDK's own without being mistaken for them. + """ + if isinstance(target, RunAggregationsProvider): + return list(target.run_aggregate_scores()) + return [] + + def _persist_with_optional_dashboard( result: AgentEvalResult, output_dir: Path, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py index 82377a118d..baa184fc5f 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py @@ -32,17 +32,18 @@ def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalRes def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: + artifacts = { + "metadata": "metadata.json", + "tasks": "tasks.jsonl", + "trials": "trials.jsonl", + "scores": "scores.jsonl", + "summary": "summary.json", + } return { "run_id": result.run_id, "output_dir": str(result.output_dir) if result.output_dir is not None else None, "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, - "artifacts": { - "metadata": "metadata.json", - "tasks": "tasks.jsonl", - "trials": "trials.jsonl", - "scores": "scores.jsonl", - "summary": "summary.json", - }, + "artifacts": artifacts, } diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 46e2c07fb6..42de6dbf62 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -13,11 +13,23 @@ from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo +from nemo_evaluator_sdk.metrics.aggregation import compute_percentiles from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name +from nemo_evaluator_sdk.values.protocol import BooleanValue, ContinuousScore from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore from pydantic import BaseModel, ConfigDict, Field +#: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, +#: discrete/count outputs, and free models (e.g. token measurements) are excluded. +_PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) + +#: Score value at or above which an attempt counts as a pass for pass@k. Full credit — pass@k answers +#: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: +#: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look +#: comparable across runs but aren't. +_PASS_VALUE = 1.0 + class AgentEvalMetricOutputCoverage(BaseModel): """Coverage counts for one metric output across scored trials.""" @@ -56,11 +68,16 @@ def from_scores( scores: Sequence[AgentEvalTaskScore], *, tasks: Sequence[AgentEvalTask] | None = None, + extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores and coverage for a set of metric scores.""" + """Build aggregated scores and coverage for a set of metric scores. + + ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced + ``runner..``), merged in so a backend's own figures are addressable the same way as ours. + """ task_list = list(tasks) if tasks is not None else None return AgentEvalSummary( - scores=_aggregate_scores(scores, task_list), + scores=_aggregate_scores(scores, task_list, extra_scores), metric_coverage=_metric_coverage(scores, task_list), task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), @@ -113,12 +130,15 @@ class AgentEvalResult(BaseModel): def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, + extra_scores: Sequence[AggregateScore] = (), ) -> AggregatedMetricResult: - """Aggregate per-metric-output and per-semantic-view values into range scores. + """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. - Each metric output becomes a score named ``.`` and each - semantic view a score named ``view.``. Failed and missing scores are - surfaced as ``nan_count`` so coverage is visible alongside the statistics. + Each metric output becomes a score named ``.``, each semantic view + ``view.``, and each score-like output additionally yields ``..pass@k`` + task-level rollups. Failed and missing scores are surfaced as ``nan_count`` so coverage is visible + alongside the statistics. ``extra_scores`` (runner-contributed, ``runner.``-namespaced) are appended + as-is. """ aggregated: list[AggregateScore] = [] @@ -143,9 +163,87 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) + aggregated.extend(_task_pass_at_k_scores(scores, tasks)) + aggregated.extend(extra_scores) + return AggregatedMetricResult(scores=aggregated) +def _pass_at_k(n: int, c: int, k: int) -> float: + """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. + + The probability that at least one of ``k`` samples drawn without replacement from ``n`` attempts + (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. + """ + if n - c < k: + return 1.0 + product = 1.0 + for i in range(n - c + 1, n + 1): + product *= 1.0 - k / i + return 1.0 - product + + +def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: + """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). + + pass@k is only meaningful for a per-attempt pass/fail signal, so labels, discrete/count outputs, + and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks + the set is empty and pass@k is skipped. + """ + scorelike: set[tuple[str, str]] = set() + if tasks is None: + return scorelike + for task in tasks: + for metric in task.metrics: + metric_type = metric_type_name(metric) + for spec in metric.output_spec(): + if issubclass(spec.value_schema, _PASS_AT_K_VALUE_SCHEMAS): + scorelike.add((metric_type, spec.name)) + return scorelike + + +def _task_pass_at_k_scores( + scores: Sequence[AgentEvalTaskScore], + tasks: Sequence[AgentEvalTask] | None, +) -> list[AggregateScore]: + """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). + + For each score-like metric output, group trials by task, count attempts ``n`` and passes ``c`` + (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as + the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` attempts). + ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. + """ + scorelike = _scorelike_outputs(tasks) + if not scorelike: + return [] + aggregated: list[AggregateScore] = [] + for metric_type, output_name in sorted(scorelike): + attempts_and_passes: dict[str, list[int]] = {} # task_id -> [n_attempts, n_passes] + for score in scores: + if score.metric_type != metric_type: + continue + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + output = _score_output(score, output_name) + value = _semantic_value(output) if output is not None else None + if value is None: + continue + counts = attempts_and_passes.setdefault(score.task_id, [0, 0]) + counts[0] += 1 + if value >= _PASS_VALUE: + counts[1] += 1 + if not attempts_and_passes: + continue + max_n = max(n for n, _ in attempts_and_passes.values()) + for k in range(1, max_n + 1): + per_task = [_pass_at_k(n, c, k) for n, c in attempts_and_passes.values() if n >= k] + if per_task: + aggregated.append( + _aggregate_range_score(f"{metric_type}.{output_name}.pass@{k}", per_task, len(per_task)) + ) + return aggregated + + def _aggregate_range_score(name: str, values: list[float], total: int) -> AggregateRangeScore: finite = [value for value in values if math.isfinite(value)] count = len(finite) @@ -154,7 +252,13 @@ def _aggregate_range_score(name: str, values: list[float], total: int) -> Aggreg return AggregateRangeScore(name=name, count=0, nan_count=nan_count) total_sum = sum(finite) mean = total_sum / count - variance = sum((value - mean) ** 2 for value in finite) / count + # Report both conventions explicitly rather than picking one: the population figures describe the + # values actually evaluated, the sample figures estimate the process they were drawn from (which is + # what repeated trials over one task are sampling). Sample stats are undefined for a single value. + sum_sq_dev = sum((value - mean) ** 2 for value in finite) + variance = sum_sq_dev / count + sample_variance = sum_sq_dev / (count - 1) if count > 1 else None + percentiles = compute_percentiles(sorted(finite)) return AggregateRangeScore( name=name, count=count, @@ -165,6 +269,14 @@ def _aggregate_range_score(name: str, values: list[float], total: int) -> Aggreg max=max(finite), variance=variance, std_dev=math.sqrt(variance), + sample_variance=sample_variance, + sample_std_dev=math.sqrt(sample_variance) if sample_variance is not None else None, + # Reuse the deterministic-metric percentile helper so agent-eval and metric aggregation report + # the same distribution the same way. + percentiles=percentiles, + # Surfaced alongside the other basic stats so `median` means the same thing whether a score + # was computed here or imported from a backend that reports one without a full distribution. + median=percentiles.p50, ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py index d13d22086e..13694103ae 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py @@ -51,6 +51,7 @@ import hashlib import json import logging +import math import os import re import signal @@ -64,6 +65,7 @@ from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor +from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore from pydantic import BaseModel, ConfigDict, Field logger = logging.getLogger(__name__) @@ -359,6 +361,16 @@ class GymAgentTaskRunner: def __init__(self, *, config: GymRuntimeConfig) -> None: self._config = config + self._run_aggregations: dict[str, Any] | None = None + + def run_aggregate_scores(self) -> Sequence[AggregateScore]: + """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.``. + + Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it + natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites + exactly the confusion the namespace is there to prevent. + """ + return _aggregate_scores_from_gym(self._run_aggregations) def runner_info(self) -> RunnerInfo: """Identify this runner and the Gym settings that shape its results. @@ -390,6 +402,7 @@ async def run_tasks( config: AgentEvalRunConfig | None = None, ) -> list[AgentEvalTrial]: cfg = self._config + self._run_aggregations = None # reset per run so a reused runner never leaks a prior run's numbers # Provenance for the log line only — the file Gym actually reads is the normalized one we # materialize below from the tasks themselves. source_dataset = _source_datasets(tasks) @@ -421,6 +434,7 @@ async def run_tasks( ) await self._run_two_step(input_path, rollouts_path, work_dir) + self._run_aggregations = _read_run_aggregations(rollouts_path) trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key) _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path) return trials @@ -729,6 +743,138 @@ def _failures_path_for(rollouts_path: Path) -> Path: return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl") +def _aggregate_metrics_path_for(rollouts_path: Path) -> Path: + """Sidecar Gym writes run-level aggregate metrics to (``_aggregate_metrics.json``).""" + return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json") + + +def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None: + """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable. + + Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` / + ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema + isn't contractual, so the SDK does not type it. + """ + path = _aggregate_metrics_path_for(rollouts_path) + if not path.exists(): + return None + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path) + return None + if not isinstance(parsed, list): + logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__) + return None + # Gym's schema is not contractual, and this runs after a collection that already succeeded — an + # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it. + aggregations: dict[str, Any] = {} + for entry in parsed: + agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None + name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None + if not isinstance(name, str): + logger.warning( + "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref + ) + continue + aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"} + return aggregations or None + + +#: Gym flattens a distribution into ``/`` keys. Its RewardProfiler emits this exact set +#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which +#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we +#: treat a group of keys as one distribution: a resources-server is free to define a metric literally +#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial +#: match would rename someone's standalone metric into a statistic of a distribution that never existed. +_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std") + +#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``). +_GYM_REDUNDANT_METRICS = frozenset({"reward"}) + + +def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]: + """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.``. + + Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the + resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries — + so the max/min/median/std that make a distribution never appear there, and every metric would arrive + as a lone ``mean/`` scalar. + + Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other + numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are + labels or notes, not measurements. + + Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the + agent adds no disambiguation, and a reader needs to know which *backend* produced a number. + """ + if not aggregations: + return [] + multi_agent = len(aggregations) > 1 + scores: list[AggregateScore] = [] + for agent_name, payload in sorted(aggregations.items()): + agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None + if not isinstance(agent_metrics, Mapping): + continue + # One agent per run is the norm, so `runner.gym.` reads cleanly; qualify by agent only + # when a run really did produce several, where the unqualified names would collide. + prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym." + scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix)) + return scores + + +def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]: + families: dict[str, dict[str, float]] = {} + scalars: dict[str, float] = {} + for key, value in agent_metrics.items(): + number = _as_float(value) + if number is None: + continue + stat, _, metric = key.partition("/") + if metric and stat in _GYM_STAT_FAMILY: + families.setdefault(metric, {})[stat] = number + else: + scalars[key] = number + + scores: list[AggregateScore] = [] + for metric, stats in sorted(families.items()): + if set(stats) < set(_GYM_STAT_FAMILY): + # Not a distribution we can vouch for; keep each key as the standalone number it may well be. + scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()}) + continue + if metric in _GYM_REDUNDANT_METRICS: + continue + scores.append( + AggregateRangeScore( + name=f"{prefix}{metric}", + # Gym reports the statistics but not the sample size behind them, and inventing one + # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated. + count=None, + nan_count=0, + mean=stats["mean"], + min=stats["min"], + max=stats["max"], + median=stats["median"], + # Gym computes this with pandas (ddof=1), so it is the sample standard deviation. + sample_std_dev=stats["std"], + sample_variance=stats["std"] ** 2, + ) + ) + scores.extend( + AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value) + for key, value in sorted(scalars.items()) + if key not in _GYM_REDUNDANT_METRICS + ) + return scores + + +def _as_float(value: Any) -> float | None: + """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements).""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) if math.isfinite(value) else None + + def _ensure_fresh_output(rollouts_path: Path) -> None: """Enforce one Gym run per output dir (the AgentEvaluator convention). diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py index 378373cc4a..d97669e2d8 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py @@ -25,6 +25,7 @@ CandidateEvidence, EvidenceDescriptor, ) +from nemo_evaluator_sdk.values.results import AggregateScore from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator @@ -141,6 +142,22 @@ def callable_identity(target: object) -> str: return f"{module}.{name}" if module else name +@runtime_checkable +class RunAggregationsProvider(Protocol): + """Optional companion to :class:`AgentTaskRunner`: a runner that computed its own run-level + aggregations (a backend's pass@k, reward profile, environment-specific metrics) exposes them here, + mapped onto the SDK's typed aggregate scores. The evaluator calls this after ``run_tasks``; + implementers stash their numbers during the run and convert them here. + + Returned scores are merged into ``summary.scores``, so a backend's own figures sit alongside the + SDK's and are addressable by name the same way. Implementers must namespace names under + ``runner..`` so an imported figure is never mistaken for one the SDK computed. Runners + with no run-level aggregations simply don't implement this protocol. + """ + + def run_aggregate_scores(self) -> Sequence[AggregateScore]: ... + + @runtime_checkable class AgentTrialSerde(Protocol): """Read/write a single stored trial artifact as an :class:`AgentEvalTrial`. diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py index 2e214fe3d3..4a626e9c1a 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/metrics/aggregation.py @@ -183,7 +183,7 @@ def _compute_percentile(sorted_values: list[float], percentile: float) -> float: return sorted_values[lower_idx] + frac * (sorted_values[lower_idx + 1] - sorted_values[lower_idx]) -def _compute_percentiles(sorted_values: list[float]) -> Percentiles: +def compute_percentiles(sorted_values: list[float]) -> Percentiles: """Compute the fixed percentile set used by SDK aggregate output. Args: @@ -335,11 +335,18 @@ def aggregate_metrics( n = len(values) mean = results.stats.mean or 0 - # Use population variance because these rows are the full evaluation set, - # not a sample intended to estimate a larger population. - variance = sum((v - mean) ** 2 for v in values) / n if n > 0 else 0 + # Report both conventions under explicit names rather than leaving the divisor implicit. + # `variance`/`stddev` stay population (divide by n): these rows are the full evaluation set, + # not a sample intended to estimate a larger population. The sample (n-1) figures are also + # provided for callers estimating the spread of the process the values were drawn from, and + # are undefined for a single value. + sum_sq_dev = sum((v - mean) ** 2 for v in values) + variance = sum_sq_dev / n if n > 0 else 0 results.stats.variance = variance results.stats.stddev = math.sqrt(variance) + sample_variance = sum_sq_dev / (n - 1) if n > 1 else None + results.stats.sample_variance = sample_variance + results.stats.sample_stddev = math.sqrt(sample_variance) if sample_variance is not None else None aggregated_scores: list[AggregateScore] = [] for score_name, metric_score in aggregated_results.items(): @@ -356,6 +363,12 @@ def aggregate_metrics( base_max = stats.max if stats.max is not None else base_mean base_variance = stats.variance if stats.variance is not None else (None if base_mean is None else 0.0) base_std_dev = stats.stddev if stats.stddev is not None else (None if base_mean is None else 0.0) + # Sample stats stay None when undefined (fewer than two values) rather than defaulting to 0.0. + base_sample_variance = stats.sample_variance + base_sample_std_dev = stats.sample_stddev + # Derived from the same helper that produces p50, so `median` and `percentiles.p50` agree + # exactly wherever both are present (rubric scores carry no percentiles but still get a median). + base_median = _compute_percentile(sorted(values), 50) if values else None if base_count == 0: base_sum = None @@ -364,6 +377,9 @@ def aggregate_metrics( base_max = None base_variance = None base_std_dev = None + base_sample_variance = None + base_sample_std_dev = None + base_median = None if has_rubric.get(score_name): rubric_dist = [ @@ -386,8 +402,11 @@ def aggregate_metrics( mean=base_mean, min=base_min, max=base_max, + median=base_median, variance=base_variance, std_dev=base_std_dev, + sample_variance=base_sample_variance, + sample_std_dev=base_sample_std_dev, rubric_distribution=rubric_dist, mode_category=mode_category, ) @@ -396,7 +415,7 @@ def aggregate_metrics( if values: # Range scores get richer distribution metadata than rubric scores. sorted_values = sorted(values) - percentiles = _compute_percentiles(sorted_values) + percentiles = compute_percentiles(sorted_values) histogram = _compute_histogram(values) else: percentiles = None @@ -411,8 +430,11 @@ def aggregate_metrics( mean=base_mean, min=base_min, max=base_max, + median=base_median, variance=base_variance, std_dev=base_std_dev, + sample_variance=base_sample_variance, + sample_std_dev=base_sample_std_dev, percentiles=percentiles, histogram=histogram, ) diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py index 03415884a0..e1d0a53837 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/values/results.py @@ -22,8 +22,11 @@ "mean", "min", "max", + "median", "std_dev", "variance", + "sample_std_dev", + "sample_variance", # Range-specific fields "score_type", "percentiles", @@ -180,6 +183,15 @@ class ScoreStats(BaseModel): default=None, description="""The population standard deviation, (note: not the sample standard deviation).""", ) + sample_variance: float | None = Field( + default=None, + description="The sample (Bessel-corrected, n-1) variance. None when fewer than two values.", + ) + sample_stddev: float | None = Field( + default=None, + description="The sample (Bessel-corrected, n-1) standard deviation, estimating the spread of the " + "process the values were drawn from. None when fewer than two values (undefined, not zero).", + ) stderr: float | None = Field(default=None, description="The standard error.") nan_count: int | None = Field( default=None, @@ -189,7 +201,9 @@ class ScoreStats(BaseModel): default=None, description="The distribution of the rubric grading criteria for the score." ) - @field_serializer("sum", "sum_squared", "min", "max", "mean", "variance", "stddev", "stderr") + @field_serializer( + "sum", "sum_squared", "min", "max", "mean", "variance", "stddev", "sample_variance", "sample_stddev", "stderr" + ) def serialize_nan(self, v: float | None) -> float | str | None: """Serialize NaN stats as string values for JSON compatibility. @@ -279,14 +293,42 @@ class AggregateScoreBase(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(description="Name of the score.") - count: int = Field(description="Number of samples evaluated (excluding NaN).") + count: int | None = Field( + default=None, + description="Number of samples evaluated (excluding NaN). None when the sample size is unknown " + "— e.g. a figure imported from a backend that reports statistics without the n behind them. " + "Distinct from 0, which asserts that nothing was evaluated.", + ) nan_count: int = Field(description="Number of samples that produced NaN scores.") sum: float | None = Field(default=None, description="Sum of all score values.") mean: float | None = Field(default=None, description="Mean score value.") min: float | None = Field(default=None, description="Minimum score value.") max: float | None = Field(default=None, description="Maximum score value.") - std_dev: float | None = Field(default=None, description="Standard deviation of the scores.") - variance: float | None = Field(default=None, description="Variance of the scores.") + median: float | None = Field( + default=None, + description="Median score value. Equal to percentiles.p50 when a percentile distribution is " + "also present; carried separately because a backend may report a median without one.", + ) + std_dev: float | None = Field( + default=None, + description="Population standard deviation of the scores (divides by n). Describes the spread of " + "the values actually evaluated. See sample_std_dev to estimate the spread of the wider process.", + ) + variance: float | None = Field( + default=None, + description="Population variance of the scores (divides by n). See sample_variance.", + ) + sample_std_dev: float | None = Field( + default=None, + description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates " + "the spread of the process the values were drawn from — the right choice when repeated trials " + "sample a stochastic system. None when fewer than two values (undefined, not zero).", + ) + sample_variance: float | None = Field( + default=None, + description="Sample variance of the scores (Bessel-corrected, divides by n-1). None when fewer " + "than two values.", + ) class AggregateRangeScore(AggregateScoreBase): @@ -339,7 +381,38 @@ def _serialize(self, handler): return data -AggregateScore = AggregateRangeScore | AggregateRubricScore +class AggregateScalarScore(AggregateScoreBase): + """A single pre-computed value with no underlying distribution available. + + For figures a backend reports as one number (e.g. an environment's own ``pass@1`` or Elo) rather + than a set of per-sample values the SDK could aggregate itself. ``value`` carries the number; + ``mean``/``min``/``max`` are left unset because there is no sample to describe. Distinct from + :class:`AggregateRangeScore` so a reader can tell "this is the whole story" from "this summarizes + ``count`` samples", instead of seeing a range score with a suspicious ``count`` of 1. + """ + + score_type: Literal["scalar"] = Field(default="scalar", description="Type of score.") + value: float = Field(description="The reported value.") + + _include_fields: frozenset[str] | None = None + + def with_fields(self, fields: frozenset[AggregateFieldName]) -> Self: + """Return a copy configured to serialize only the specified fields.""" + copy = self.model_copy() + object.__setattr__(copy, "_include_fields", {*fields, "name", "count"}) + return copy + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self._include_fields is not None: + # Always include required fields (name, count, value), plus requested fields + fields_to_include = self._include_fields | {"name", "count", "value"} + return {k: v for k, v in data.items() if k in fields_to_include} + return data + + +AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore class AggregatedMetricResult(BaseModel): diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_dashboard.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_dashboard.py index 1ce0929322..f500c24730 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_dashboard.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_dashboard.py @@ -5,7 +5,7 @@ from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_evaluator_sdk.metrics.protocol import MetricOutput -from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore +from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScalarScore def test_dashboard_contains_metric_rollups_and_outputs() -> None: @@ -44,3 +44,25 @@ def test_dashboard_contains_metric_rollups_and_outputs() -> None: assert "trial-1" in html assert "partial" in html assert "Scores" in html + + +def test_dashboard_renders_a_runner_imported_scalar_as_its_value_not_a_blank_mean() -> None: + # A scalar has no distribution, so reading the table straight off `mean` would leave the one number + # worth seeing blank, and print a sample size of 0 as though the metric had failed everywhere. + result = AgentEvalResult( + run_id="run-1", + tasks=[], + trials=[], + scores=[], + summary=AgentEvalSummary( + scores=AggregatedMetricResult( + scores=[AggregateScalarScore(name="runner.gym.arena_elo/score", count=None, nan_count=0, value=1523.0)] + ), + ), + ) + + html = render_dashboard(result) + + assert "1523.000" in html + assert "runner.gym.arena_elo/score" in html + assert "—" in html # count is "not reported" (None), not a zero sample size diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py new file mode 100644 index 0000000000..b0df392f70 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_gym_aggregate_scores.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Mapping Gym's flattened ``key_metrics`` onto typed aggregate scores.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import _aggregate_scores_from_gym +from nemo_evaluator_sdk.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore + + +def _gym(agent_metrics: Mapping[str, object], agent: str = "simple_agent") -> dict[str, object]: + """A Gym aggregate-metrics payload. Imports read ``agent_metrics`` (the full run-level set), not + ``key_metrics`` — which by default holds only the ``mean/*`` subset.""" + return {agent: {"agent_metrics": agent_metrics}} + + +def _by_name(scores: Sequence[AggregateScore]) -> dict[str, AggregateScore]: + return {score.name: score for score in scores} + + +def test_a_full_stat_family_is_reassembled_into_one_range_score() -> None: + scores = _by_name( + _aggregate_scores_from_gym( + _gym( + { + "mean/total_tokens": 120.0, + "max/total_tokens": 200.0, + "min/total_tokens": 50.0, + "median/total_tokens": 110.0, + "std/total_tokens": 12.0, + } + ) + ) + ) + + assert list(scores) == ["runner.gym.total_tokens"] + score = scores["runner.gym.total_tokens"] + assert isinstance(score, AggregateRangeScore) + assert (score.mean, score.min, score.max, score.median) == (120.0, 50.0, 200.0, 110.0) + # Gym computes std with pandas (ddof=1), so it lands on the sample field, not the population one. + assert score.sample_std_dev == 12.0 + assert score.std_dev is None + + +def test_a_partial_stat_family_is_left_as_standalone_scalars() -> None: + # A resources-server may define a metric literally named `mean`; re-assembling on a partial match + # would rename someone's real metric into a statistic of a distribution that never existed. + scores = _by_name(_aggregate_scores_from_gym(_gym({"mean/accuracy": 0.8, "max/accuracy": 1.0}))) + + assert set(scores) == {"runner.gym.mean/accuracy", "runner.gym.max/accuracy"} + assert all(isinstance(score, AggregateScalarScore) for score in scores.values()) + mean_accuracy = scores["runner.gym.mean/accuracy"] + assert isinstance(mean_accuracy, AggregateScalarScore) and mean_accuracy.value == 0.8 + + +def test_environment_specific_metrics_survive_as_scalars() -> None: + # 36 of Gym's ~97 resources-servers override compute_metrics, emitting keys in their own shapes. + scores = _by_name(_aggregate_scores_from_gym(_gym({"arena_elo/score": 1523.0, "easy/pass@1/accuracy": 0.42}))) + + assert set(scores) == {"runner.gym.arena_elo/score", "runner.gym.easy/pass@1/accuracy"} + easy = scores["runner.gym.easy/pass@1/accuracy"] + assert isinstance(easy, AggregateScalarScore) and easy.value == 0.42 + + +def test_reward_is_skipped_because_the_sdk_scores_it_natively() -> None: + scores = _by_name( + _aggregate_scores_from_gym( + _gym( + { + "mean/reward": 0.6, + "max/reward": 1.0, + "min/reward": 0.0, + "median/reward": 0.5, + "std/reward": 0.4, + "mean/steps": 3.0, + "max/steps": 5.0, + "min/steps": 1.0, + "median/steps": 3.0, + "std/steps": 1.2, + } + ) + ) + ) + + # gym_reward.reward already carries this, derived from the same rollouts. + assert list(scores) == ["runner.gym.steps"] + + +def test_nothing_numeric_is_dropped_from_a_custom_environment_payload() -> None: + """Every numeric key Gym reported must be represented — as a re-assembled family or as a scalar. + + The invariant that matters: importing is a *renaming*, never a filter. Only ``reward`` (redundant + by construction) and non-numeric values are allowed to disappear. + """ + key_metrics = { + "mean/latency_s": 1.0, + "max/latency_s": 2.0, + "min/latency_s": 0.5, + "median/latency_s": 0.9, + "std/latency_s": 0.3, + "mean/partial": 0.9, # incomplete family + "arena_elo/score": 1200.0, + "easy/pass@1/accuracy": 0.5, + "hard/pass@1/accuracy": 0.1, + "num_rollouts": 40, + "notes": "not a measurement", # non-numeric: stays only in the opaque payload + "converged": True, # a bool is a flag, not a measurement + } + scores = _aggregate_scores_from_gym(_gym(key_metrics)) + + numeric_keys = {key for key, value in key_metrics.items() if isinstance(value, (int, float)) and value is not True} + represented = set() + for score in scores: + name = score.name.removeprefix("runner.gym.") + if isinstance(score, AggregateRangeScore): + represented.update(f"{stat}/{name}" for stat in ("mean", "max", "min", "median", "std")) + else: + represented.add(name) + + assert represented == numeric_keys + + +def test_names_are_qualified_by_agent_only_when_a_run_produced_several() -> None: + # One agent per run is the norm, so the agent name adds nothing; with two it prevents a collision. + one = _by_name(_aggregate_scores_from_gym(_gym({"score": 1.0}))) + assert list(one) == ["runner.gym.score"] + + two = _by_name( + _aggregate_scores_from_gym({"a": {"agent_metrics": {"score": 1.0}}, "b": {"agent_metrics": {"score": 2.0}}}) + ) + assert set(two) == {"runner.gym.a.score", "runner.gym.b.score"} + + +def test_absent_or_malformed_aggregations_yield_nothing() -> None: + assert _aggregate_scores_from_gym(None) == [] + assert _aggregate_scores_from_gym({}) == [] + assert _aggregate_scores_from_gym({"agent": {"group_level_metrics": []}}) == [] # no agent_metrics + assert _aggregate_scores_from_gym({"agent": "not a mapping"}) == [] + + +def test_imported_scores_report_an_unknown_sample_size_rather_than_zero() -> None: + # Gym reports statistics without the n behind them. count=0 would assert that nothing was + # evaluated — false, and a landmine for anything that divides by it. + scores = _aggregate_scores_from_gym( + _gym( + { + "mean/steps": 3.0, + "max/steps": 5.0, + "min/steps": 1.0, + "median/steps": 3.0, + "std/steps": 1.2, + "elo": 1200.0, + } + ) + ) + + assert scores + assert all(score.count is None for score in scores) + + +def test_imports_read_agent_metrics_not_the_key_metrics_subset() -> None: + """Reading ``key_metrics`` would silently degrade every distribution into a lone mean. + + Gym's ``get_key_metrics`` defaults to selecting only the ``mean/*`` entries of ``agent_metrics``, so + a payload's ``key_metrics`` never carries the max/min/median/std that make a stat family. Sourcing + from it would leave every metric as a scalar named ``runner.gym.mean/``. + """ + payload = { + "simple_agent": { + "agent_metrics": { + "mean/steps": 3.0, + "max/steps": 5.0, + "min/steps": 1.0, + "median/steps": 3.0, + "std/steps": 1.2, + }, + # what Gym's default get_key_metrics would select — a strict subset, means only + "key_metrics": {"mean/steps": 3.0}, + } + } + + scores = _by_name(_aggregate_scores_from_gym(payload)) + + assert list(scores) == ["runner.gym.steps"] + steps = scores["runner.gym.steps"] + assert isinstance(steps, AggregateRangeScore) + assert steps.max == 5.0 # would be unreachable if key_metrics were the source + + +def test_median_matches_p50_wherever_both_are_reported() -> None: + """`median` is documented as equal to `percentiles.p50`, so the two must not drift. + + They are produced by different call sites (agent-eval's `_aggregate_range_score` and the + deterministic-metric `aggregate_metrics`), which is exactly how one silently stops matching. + """ + from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary + from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore + from nemo_evaluator_sdk.metrics.protocol import MetricOutput + + scores = [ + AgentEvalTaskScore( + id=f"r:t{i}:tr:m", + run_id="r", + task_id=f"t{i}", + trial_id=f"tr{i}", + metric_type="m", + status=AgentEvalScoreStatus.COMPLETED, + outputs=[MetricOutput(name="score", value=value)], + ) + for i, value in enumerate([0.1, 0.4, 0.9, 0.2]) + ] + + summary = AgentEvalSummary.from_scores(scores) + aggregate = next(s for s in summary.scores.scores if s.name == "m.score") + + assert isinstance(aggregate, AggregateRangeScore) + assert aggregate.percentiles is not None + assert aggregate.median == aggregate.percentiles.p50 diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py new file mode 100644 index 0000000000..e2988e2411 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_pass_at_k.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task-level pass@k aggregation: the unbiased estimator, output gating, threshold, and uniformity +across metric types (so Gym and Harbor trials aggregate identically).""" + +from __future__ import annotations + +import pytest +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary, _pass_at_k +from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask +from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricResult +from nemo_evaluator_sdk.values.protocol import MetricOutputSpec + + +class _ScoreMetric: + """Minimal metric declaring a single continuous-score output (pass@k-eligible).""" + + def __init__(self, metric_type: str) -> None: + self._type = metric_type + + @property + def type(self) -> str: + return self._type + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.continuous_score("reward")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: # pragma: no cover - not exercised + raise NotImplementedError + + +class _LabelMetric: + """Minimal metric declaring a label output (must NOT get pass@k).""" + + @property + def type(self) -> str: + return "verdict" + + def output_spec(self) -> list[MetricOutputSpec]: + return [MetricOutputSpec.label("category")] + + async def compute_scores(self, input: MetricInput) -> MetricResult: # pragma: no cover - not exercised + raise NotImplementedError + + +def _task(task_id: str, *metrics: Metric) -> AgentEvalTask: + return AgentEvalTask(id=task_id, intent="t", inputs={}, metrics=list(metrics)) + + +def _score(task_id: str, trial_id: str, metric_type: str, name: str, value: object) -> AgentEvalTaskScore: + return AgentEvalTaskScore( + id=f"{task_id}:{trial_id}:{metric_type}", + run_id="run", + task_id=task_id, + trial_id=trial_id, + metric_type=metric_type, + status=AgentEvalScoreStatus.COMPLETED, + outputs=[MetricOutput(name=name, value=value)], + ) + + +def test_pass_at_k_unbiased_estimator() -> None: + assert _pass_at_k(2, 0, 1) == 0.0 # no passes + assert _pass_at_k(2, 2, 1) == 1.0 # all pass + assert _pass_at_k(2, 1, 1) == pytest.approx(0.5) + assert _pass_at_k(2, 1, 2) == 1.0 # n-c < k -> guaranteed hit + assert _pass_at_k(3, 1, 1) == pytest.approx(1 / 3) + assert _pass_at_k(4, 2, 2) == pytest.approx(5 / 6) # 1 - C(2,2)/C(4,2) + + +def test_task_pass_at_k_gated_and_uniform_across_metric_types() -> None: + # Two tasks x 2 attempts, scored by two reward metric types (mimicking Gym + Harbor) plus a label + # metric. t1 rewards [1.0, 0.0] (1/2 pass); t2 [1.0, 1.0] (2/2). pass@1 = mean(0.5, 1.0) = 0.75. + gym, harbor, label = _ScoreMetric("gym_reward"), _ScoreMetric("harbor_reward"), _LabelMetric() + tasks = [_task("t1", gym, harbor, label), _task("t2", gym, harbor, label)] + scores: list[AgentEvalTaskScore] = [] + for mt in ("gym_reward", "harbor_reward"): + scores += [ + _score("t1", "a0", mt, "reward", 1.0), + _score("t1", "a1", mt, "reward", 0.0), + _score("t2", "a0", mt, "reward", 1.0), + _score("t2", "a1", mt, "reward", 1.0), + ] + scores.append(_score("t1", "a0", "verdict", "category", "good")) # label — ignored by pass@k + + summary = AgentEvalSummary.from_scores(scores, tasks=tasks) + by_name = {score.name: score for score in summary.scores.scores} + + for metric_type in ("gym_reward", "harbor_reward"): # uniform across runners + assert by_name[f"{metric_type}.reward.pass@1"].mean == pytest.approx(0.75) + assert by_name[f"{metric_type}.reward.pass@2"].mean == pytest.approx(1.0) + assert not any(name.startswith("verdict") and ".pass@" in name for name in by_name) # label not eligible + + +def test_partial_credit_is_not_a_pass() -> None: + # pass@k answers "did the agent solve the task", so only full credit counts: of attempts 0.5 and + # 1.0, exactly one is a pass. + tasks = [_task("t1", _ScoreMetric("reward"))] + scores = [_score("t1", "a0", "reward", "reward", 0.5), _score("t1", "a1", "reward", "reward", 1.0)] + + by_name = {s.name: s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores} + + assert by_name["reward.reward.pass@1"].mean == pytest.approx(0.5) + assert by_name["reward.reward.pass@2"].mean == pytest.approx(1.0) # one of the two attempts passed + + +def test_population_and_sample_stats_are_both_reported() -> None: + # The two conventions answer different questions, so both are named explicitly rather than + # leaving the divisor implicit: population divides by n, sample (Bessel) by n-1. + tasks = [_task("t1", _ScoreMetric("reward")), _task("t2", _ScoreMetric("reward"))] + values = [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0] # n=10, mean=0.6, sum_sq_dev=2.4 + scores = [_score(f"t{1 + index % 2}", f"a{index}", "reward", "reward", value) for index, value in enumerate(values)] + + aggregate = next( + s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward" + ) + + assert aggregate.count == 10 + assert aggregate.mean == pytest.approx(0.6) + assert aggregate.std_dev == pytest.approx((2.4 / 10) ** 0.5) # population + assert aggregate.sample_std_dev == pytest.approx((2.4 / 9) ** 0.5) # sample (Bessel-corrected) + assert aggregate.variance == pytest.approx(2.4 / 10) + assert aggregate.sample_variance == pytest.approx(2.4 / 9) + + +def test_sample_stats_undefined_for_a_single_value() -> None: + # Sample statistics need at least two observations; None means "not estimable", not zero. + tasks = [_task("t1", _ScoreMetric("reward"))] + scores = [_score("t1", "a0", "reward", "reward", 1.0)] + + aggregate = next( + s for s in AgentEvalSummary.from_scores(scores, tasks=tasks).scores.scores if s.name == "reward.reward" + ) + + assert aggregate.std_dev == 0.0 # population is well-defined for one value + assert aggregate.sample_std_dev is None + assert aggregate.sample_variance is None + + +def test_pass_at_k_skipped_without_task_specs() -> None: + # No tasks -> no metric specs -> pass@k can't know which outputs are score-like, so none emitted. + scores = [_score("t1", "a0", "reward", "reward", 1.0)] + summary = AgentEvalSummary.from_scores(scores, tasks=None) + assert not any(".pass@" in score.name for score in summary.scores.scores) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_runner_aggregations.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_runner_aggregations.py new file mode 100644 index 0000000000..7b45b12718 --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_runner_aggregations.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The evaluator merges run-level aggregations only from runners that opt into +:class:`RunAggregationsProvider`, and only under the runner's own namespace.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from nemo_evaluator_sdk.agent_eval.evaluator import _collect_runner_aggregate_scores +from nemo_evaluator_sdk.agent_eval.results import AgentEvalSummary +from nemo_evaluator_sdk.values.results import AggregateScalarScore, AggregateScore + + +class _Provider: + def run_aggregate_scores(self) -> Sequence[AggregateScore]: + return [AggregateScalarScore(name="runner.gym.pass@1", count=None, nan_count=0, value=0.6)] + + +class _EmptyProvider: + def run_aggregate_scores(self) -> Sequence[AggregateScore]: + return [] + + +class _PlainRunner: + """A runner that does not implement RunAggregationsProvider.""" + + +def test_collect_empty_when_the_provider_had_nothing_to_map() -> None: + assert _collect_runner_aggregate_scores(_EmptyProvider()) == [] + + +def test_collect_empty_when_runner_does_not_implement_protocol() -> None: + assert _collect_runner_aggregate_scores(_PlainRunner()) == [] + + +def test_runner_scores_are_merged_into_the_summary_under_their_own_namespace() -> None: + # Merging (rather than parking them in a separate untyped bag) is the point: a backend's own + # numbers become addressable by name the same way ours are, while the `runner.` prefix keeps it + # obvious they were imported rather than computed here. + scores = _collect_runner_aggregate_scores(_Provider()) + summary = AgentEvalSummary.from_scores([], tasks=[], extra_scores=scores) + + by_name = {score.name: score for score in summary.scores.scores} + imported = by_name["runner.gym.pass@1"] + assert isinstance(imported, AggregateScalarScore) and imported.value == 0.6 + assert all(name.startswith("runner.") for name in by_name) diff --git a/packages/nemo_evaluator_sdk/tests/test_api.py b/packages/nemo_evaluator_sdk/tests/test_api.py index 84cb8a0365..bc7eecb38b 100644 --- a/packages/nemo_evaluator_sdk/tests/test_api.py +++ b/packages/nemo_evaluator_sdk/tests/test_api.py @@ -19,6 +19,7 @@ Percentiles, RowScore, RubricScoreStat, + ScoreStats, ) from pytest_mock import MockerFixture @@ -395,8 +396,11 @@ def test_to_records_aggregate_includes_mode_category_and_histogram_json(self): "mean": 1.5, "min": 1.0, "max": 2.0, + "median": None, "std_dev": 0.5, "variance": 0.25, + "sample_std_dev": None, + "sample_variance": None, "score_type": "rubric", "rubric_distribution": [ {"label": "good", "description": None, "value": 2, "count": 1}, @@ -471,3 +475,33 @@ def test_format_summary_without_preview_or_aggregates(self): assert "(no rows)" in formatted assert "Row preview" not in formatted assert "Error details (1 of 1 failed rows)" in formatted + + +def test_score_stats_serializes_every_nan_statistic_as_a_string() -> None: + """NaN is not valid JSON, so ScoreStats renders it as "NaN" — including the sample statistics. + + sample_variance/sample_stddev were added alongside the population pair; leaving them off the + serializer would emit a bare NaN token where every sibling stat emits a quoted string. + """ + nan = float("nan") + stats = ScoreStats( + count=1, + sum=nan, + sum_squared=nan, + min=nan, + max=nan, + mean=nan, + variance=nan, + stddev=nan, + sample_variance=nan, + sample_stddev=nan, + stderr=nan, + ) + + payload = stats.model_dump() + counts_and_lists = {"count", "nan_count", "rubric_distribution"} + numeric = {key: value for key, value in payload.items() if key not in counts_and_lists} + assert numeric == dict.fromkeys(numeric, "NaN"), f"unserialized NaN in: {numeric}" + + # And the result is actually JSON-encodable, which a bare NaN would not be under strict mode. + json.dumps(payload, allow_nan=False) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py index c537ce2274..2487c10ab1 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/dashboard.py @@ -12,6 +12,7 @@ from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalTaskScore +from nemo_platform.beta.evaluator.values.results import AggregateScalarScore, AggregateScore from pydantic import BaseModel @@ -83,18 +84,41 @@ def _metric_rollups(result: AgentEvalResult) -> str: rows.append( "" f"{_e(score.name)}" - f"{_format_score(score.mean)}" - f"{_e(score.count)}" + f"{_format_score(_headline_value(score))}" + f"{_format_score(_median(score))}" + f"{_format_score(score.sample_std_dev)}" + f"{_count(score.count)}" f"{_e(score.nan_count)}" "" ) return ( - "" - + "".join(rows) - + "
NameMeanCountNaN
" + "" + "" + "".join(rows) + "
NameValueMedianStd devCountNaN
" ) +def _headline_value(score: AggregateScore) -> float | None: + """The one number to show: a scalar's ``value``, otherwise the mean of the distribution. + + A scalar score has no mean — rendering the column straight off ``score.mean`` would leave every + runner-imported figure blank in the table where it is the only thing worth reading. + """ + return score.value if isinstance(score, AggregateScalarScore) else score.mean + + +def _median(score: AggregateScore) -> float | None: + percentiles = getattr(score, "percentiles", None) + return percentiles.p50 if percentiles is not None else None + + +def _count(count: int | None) -> str: + """Sample size, or an em dash when the producer didn't report one (imported aggregates). + + Tests for None specifically: a genuine 0 means every sample was NaN, which is worth seeing. + """ + return "—" if count is None else _e(count) + + def _score_table(scores: list[AgentEvalTaskScore]) -> str: if not scores: return '

No metric scores.

' diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index 8bb80c814f..697fc6f2b5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -37,6 +37,7 @@ AgentEvalTrialStatus, AgentOutput, AgentTaskRunner, + RunAggregationsProvider, RunnerInfo, ) from nemo_platform.beta.evaluator.agent_inference import ( @@ -60,6 +61,7 @@ RunConfigOnline, RunConfigOnlineModel, ) +from nemo_platform.beta.evaluator.values.results import AggregateScore from nemo_platform.beta.evaluator.values.evidence import ( EVIDENCE_FORMAT_JSON, EVIDENCE_TRACE, @@ -176,6 +178,7 @@ async def run( config=runtime_config, run_id=run_id, ) + runner_scores = _collect_runner_aggregate_scores(target) if target is not None else [] finished_at = datetime.now(UTC) metadata = RunMetadata( labels=dict(runtime_config.labels), @@ -190,7 +193,7 @@ async def run( tasks=task_list, trials=trial_list, scores=scores, - summary=AgentEvalSummary.from_scores(scores, tasks=task_list), + summary=AgentEvalSummary.from_scores(scores, tasks=task_list, extra_scores=runner_scores), metadata=metadata, ) @@ -732,6 +735,17 @@ def _describe_target( return target.runner_info() +def _collect_runner_aggregate_scores(target: object) -> list[AggregateScore]: + """The typed subset of a runner's own aggregations, for merging into ``summary.scores``. + + A runner that maps its numbers onto aggregate scores namespaces them under ``runner..``, so + they sit alongside the SDK's own without being mistaken for them. + """ + if isinstance(target, RunAggregationsProvider): + return list(target.run_aggregate_scores()) + return [] + + def _persist_with_optional_dashboard( result: AgentEvalResult, output_dir: Path, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py index d5eb90c9fa..ac152bf09a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py @@ -32,17 +32,18 @@ def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalRes def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: + artifacts = { + "metadata": "metadata.json", + "tasks": "tasks.jsonl", + "trials": "trials.jsonl", + "scores": "scores.jsonl", + "summary": "summary.json", + } return { "run_id": result.run_id, "output_dir": str(result.output_dir) if result.output_dir is not None else None, "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, - "artifacts": { - "metadata": "metadata.json", - "tasks": "tasks.jsonl", - "trials": "trials.jsonl", - "scores": "scores.jsonl", - "summary": "summary.json", - }, + "artifacts": artifacts, } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index ca142bc795..d99c0a173c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -13,11 +13,23 @@ from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo +from nemo_platform.beta.evaluator.metrics.aggregation import compute_percentiles from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name +from nemo_platform.beta.evaluator.values.protocol import BooleanValue, ContinuousScore from nemo_platform.beta.evaluator.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore from pydantic import BaseModel, ConfigDict, Field +#: Metric-output value schemas eligible for pass@k (a per-attempt "did it pass?" signal). Labels, +#: discrete/count outputs, and free models (e.g. token measurements) are excluded. +_PASS_AT_K_VALUE_SCHEMAS = (ContinuousScore, BooleanValue) + +#: Score value at or above which an attempt counts as a pass for pass@k. Full credit — pass@k answers +#: "did the agent solve the task", so partial credit is not a pass. Deliberately not configurable: +#: it's a reporting-time interpretation, and making it tunable would yield pass@k numbers that look +#: comparable across runs but aren't. +_PASS_VALUE = 1.0 + class AgentEvalMetricOutputCoverage(BaseModel): """Coverage counts for one metric output across scored trials.""" @@ -56,11 +68,16 @@ def from_scores( scores: Sequence[AgentEvalTaskScore], *, tasks: Sequence[AgentEvalTask] | None = None, + extra_scores: Sequence[AggregateScore] = (), ) -> AgentEvalSummary: - """Build aggregated scores and coverage for a set of metric scores.""" + """Build aggregated scores and coverage for a set of metric scores. + + ``extra_scores`` are already-aggregated scores contributed by the runner (namespaced + ``runner..``), merged in so a backend's own figures are addressable the same way as ours. + """ task_list = list(tasks) if tasks is not None else None return AgentEvalSummary( - scores=_aggregate_scores(scores, task_list), + scores=_aggregate_scores(scores, task_list, extra_scores), metric_coverage=_metric_coverage(scores, task_list), task_count=len(task_list) if task_list is not None else len({score.task_id for score in scores}), trial_count=len({score.trial_id for score in scores}), @@ -113,12 +130,15 @@ class AgentEvalResult(BaseModel): def _aggregate_scores( scores: Sequence[AgentEvalTaskScore], tasks: Sequence[AgentEvalTask] | None, + extra_scores: Sequence[AggregateScore] = (), ) -> AggregatedMetricResult: - """Aggregate per-metric-output and per-semantic-view values into range scores. + """Aggregate per-metric-output, per-semantic-view, and task-level pass@k values into range scores. - Each metric output becomes a score named ``.`` and each - semantic view a score named ``view.``. Failed and missing scores are - surfaced as ``nan_count`` so coverage is visible alongside the statistics. + Each metric output becomes a score named ``.``, each semantic view + ``view.``, and each score-like output additionally yields ``..pass@k`` + task-level rollups. Failed and missing scores are surfaced as ``nan_count`` so coverage is visible + alongside the statistics. ``extra_scores`` (runner-contributed, ``runner.``-namespaced) are appended + as-is. """ aggregated: list[AggregateScore] = [] @@ -143,9 +163,87 @@ def _aggregate_scores( for view_name, (values, total) in sorted(_semantic_view_values(scores, tasks).items()): aggregated.append(_aggregate_range_score(f"view.{view_name}", values, total)) + aggregated.extend(_task_pass_at_k_scores(scores, tasks)) + aggregated.extend(extra_scores) + return AggregatedMetricResult(scores=aggregated) +def _pass_at_k(n: int, c: int, k: int) -> float: + """Unbiased pass@k estimator (Chen et al., 2021): ``1 - C(n-c, k) / C(n, k)``. + + The probability that at least one of ``k`` samples drawn without replacement from ``n`` attempts + (``c`` of them passing) is a pass. Caller guarantees ``1 <= k <= n``. + """ + if n - c < k: + return 1.0 + product = 1.0 + for i in range(n - c + 1, n + 1): + product *= 1.0 - k / i + return 1.0 - product + + +def _scorelike_outputs(tasks: Sequence[AgentEvalTask] | None) -> set[tuple[str, str]]: + """``(metric_type, output_name)`` pairs whose declared value is a score (continuous or boolean). + + pass@k is only meaningful for a per-attempt pass/fail signal, so labels, discrete/count outputs, + and free models (e.g. token measurements) are excluded. Needs task metric specs; with no tasks + the set is empty and pass@k is skipped. + """ + scorelike: set[tuple[str, str]] = set() + if tasks is None: + return scorelike + for task in tasks: + for metric in task.metrics: + metric_type = metric_type_name(metric) + for spec in metric.output_spec(): + if issubclass(spec.value_schema, _PASS_AT_K_VALUE_SCHEMAS): + scorelike.add((metric_type, spec.name)) + return scorelike + + +def _task_pass_at_k_scores( + scores: Sequence[AgentEvalTaskScore], + tasks: Sequence[AgentEvalTask] | None, +) -> list[AggregateScore]: + """Task-level pass@k over the R trials per task, aggregated across tasks (uniform for any runner). + + For each score-like metric output, group trials by task, count attempts ``n`` and passes ``c`` + (value ``>= _PASS_VALUE``), then emit ``..pass@k`` for ``k`` in ``1..max(n)`` as + the across-task mean of the unbiased per-task estimator (over tasks with at least ``k`` attempts). + ``pass@1`` equals the macro per-task pass rate, i.e. the task-level mean. + """ + scorelike = _scorelike_outputs(tasks) + if not scorelike: + return [] + aggregated: list[AggregateScore] = [] + for metric_type, output_name in sorted(scorelike): + attempts_and_passes: dict[str, list[int]] = {} # task_id -> [n_attempts, n_passes] + for score in scores: + if score.metric_type != metric_type: + continue + if score.status not in (AgentEvalScoreStatus.COMPLETED, AgentEvalScoreStatus.PARTIAL): + continue + output = _score_output(score, output_name) + value = _semantic_value(output) if output is not None else None + if value is None: + continue + counts = attempts_and_passes.setdefault(score.task_id, [0, 0]) + counts[0] += 1 + if value >= _PASS_VALUE: + counts[1] += 1 + if not attempts_and_passes: + continue + max_n = max(n for n, _ in attempts_and_passes.values()) + for k in range(1, max_n + 1): + per_task = [_pass_at_k(n, c, k) for n, c in attempts_and_passes.values() if n >= k] + if per_task: + aggregated.append( + _aggregate_range_score(f"{metric_type}.{output_name}.pass@{k}", per_task, len(per_task)) + ) + return aggregated + + def _aggregate_range_score(name: str, values: list[float], total: int) -> AggregateRangeScore: finite = [value for value in values if math.isfinite(value)] count = len(finite) @@ -154,7 +252,13 @@ def _aggregate_range_score(name: str, values: list[float], total: int) -> Aggreg return AggregateRangeScore(name=name, count=0, nan_count=nan_count) total_sum = sum(finite) mean = total_sum / count - variance = sum((value - mean) ** 2 for value in finite) / count + # Report both conventions explicitly rather than picking one: the population figures describe the + # values actually evaluated, the sample figures estimate the process they were drawn from (which is + # what repeated trials over one task are sampling). Sample stats are undefined for a single value. + sum_sq_dev = sum((value - mean) ** 2 for value in finite) + variance = sum_sq_dev / count + sample_variance = sum_sq_dev / (count - 1) if count > 1 else None + percentiles = compute_percentiles(sorted(finite)) return AggregateRangeScore( name=name, count=count, @@ -165,6 +269,14 @@ def _aggregate_range_score(name: str, values: list[float], total: int) -> Aggreg max=max(finite), variance=variance, std_dev=math.sqrt(variance), + sample_variance=sample_variance, + sample_std_dev=math.sqrt(sample_variance) if sample_variance is not None else None, + # Reuse the deterministic-metric percentile helper so agent-eval and metric aggregation report + # the same distribution the same way. + percentiles=percentiles, + # Surfaced alongside the other basic stats so `median` means the same thing whether a score + # was computed here or imported from a backend that reports one without a full distribution. + median=percentiles.p50, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py index ef7f75ccde..7fc81ec406 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py @@ -51,6 +51,7 @@ import hashlib import json import logging +import math import os import re import signal @@ -64,6 +65,7 @@ from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor +from nemo_platform.beta.evaluator.values.results import AggregateRangeScore, AggregateScalarScore, AggregateScore from pydantic import BaseModel, ConfigDict, Field logger = logging.getLogger(__name__) @@ -359,6 +361,16 @@ class GymAgentTaskRunner: def __init__(self, *, config: GymRuntimeConfig) -> None: self._config = config + self._run_aggregations: dict[str, Any] | None = None + + def run_aggregate_scores(self) -> Sequence[AggregateScore]: + """Gym's ``agent_metrics`` mapped onto typed aggregate scores, namespaced ``runner.gym.``. + + Satisfies :class:`RunAggregationsProvider`. ``reward`` is skipped: the SDK already scores it + natively as ``gym_reward.reward``, and two differently-derived numbers under one name invites + exactly the confusion the namespace is there to prevent. + """ + return _aggregate_scores_from_gym(self._run_aggregations) def runner_info(self) -> RunnerInfo: """Identify this runner and the Gym settings that shape its results. @@ -390,6 +402,7 @@ async def run_tasks( config: AgentEvalRunConfig | None = None, ) -> list[AgentEvalTrial]: cfg = self._config + self._run_aggregations = None # reset per run so a reused runner never leaks a prior run's numbers # Provenance for the log line only — the file Gym actually reads is the normalized one we # materialize below from the tasks themselves. source_dataset = _source_datasets(tasks) @@ -421,6 +434,7 @@ async def run_tasks( ) await self._run_two_step(input_path, rollouts_path, work_dir) + self._run_aggregations = _read_run_aggregations(rollouts_path) trials = _trials_from_rollouts(rollouts_path, tasks, index_to_task_id, reward_key=cfg.reward_key) _require_full_coverage(tasks, covered_task_ids={trial.task_id for trial in trials}, rollouts_path=rollouts_path) return trials @@ -729,6 +743,138 @@ def _failures_path_for(rollouts_path: Path) -> Path: return rollouts_path.with_name(rollouts_path.stem + "_failures.jsonl") +def _aggregate_metrics_path_for(rollouts_path: Path) -> Path: + """Sidecar Gym writes run-level aggregate metrics to (``_aggregate_metrics.json``).""" + return rollouts_path.with_name(rollouts_path.stem + "_aggregate_metrics.json") + + +def _read_run_aggregations(rollouts_path: Path) -> dict[str, Any] | None: + """Parse Gym's ``rollouts_aggregate_metrics.json``, or ``None`` when absent/unparseable. + + Gym's file is a list with one entry per agent (``agent_ref`` / ``agent_metrics`` / ``key_metrics`` / + ``group_level_metrics``), so it is returned keyed by agent name. Carried through as-is — Gym's schema + isn't contractual, so the SDK does not type it. + """ + path = _aggregate_metrics_path_for(rollouts_path) + if not path.exists(): + return None + try: + parsed = json.loads(path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + logger.warning("Could not parse Gym aggregate metrics at %s; skipping run aggregations.", path) + return None + if not isinstance(parsed, list): + logger.warning("Unexpected Gym aggregate-metrics shape at %s (%s); skipping.", path, type(parsed).__name__) + return None + # Gym's schema is not contractual, and this runs after a collection that already succeeded — an + # entry in an unexpected shape must not raise out of run_tasks and discard every trial with it. + aggregations: dict[str, Any] = {} + for entry in parsed: + agent_ref = entry.get("agent_ref") if isinstance(entry, Mapping) else None + name = agent_ref.get("name") if isinstance(agent_ref, Mapping) else None + if not isinstance(name, str): + logger.warning( + "Skipping a Gym aggregate-metrics entry in %s with no usable agent_ref.name (got %r).", path, agent_ref + ) + continue + aggregations[name] = {key: value for key, value in entry.items() if key != "agent_ref"} + return aggregations or None + + +#: Gym flattens a distribution into ``/`` keys. Its RewardProfiler emits this exact set +#: together for every numeric column (``describe_dataframe``), minus ``histogram``, which +#: ``prepare_for_serialization`` strips before the file is written. All five must be present before we +#: treat a group of keys as one distribution: a resources-server is free to define a metric literally +#: named ``mean`` (36 of Gym's ~97 servers override ``compute_metrics``), and re-assembling on a partial +#: match would rename someone's standalone metric into a statistic of a distribution that never existed. +_GYM_STAT_FAMILY = ("mean", "max", "min", "median", "std") + +#: Metric Gym reports that the SDK already computes natively from the same rollouts (``gym_reward.reward``). +_GYM_REDUNDANT_METRICS = frozenset({"reward"}) + + +def _aggregate_scores_from_gym(aggregations: Mapping[str, Any] | None) -> list[AggregateScore]: + """Map Gym's run-level ``agent_metrics`` onto typed aggregate scores named ``runner.gym.``. + + Reads ``agent_metrics``, not ``key_metrics``: ``key_metrics`` is a *subset* of it chosen by the + resources-server, and the default selection (``get_key_metrics``) keeps only the ``mean/*`` entries — + so the max/min/median/std that make a distribution never appear there, and every metric would arrive + as a lone ``mean/`` scalar. + + Keys forming a full stat family are re-assembled into one :class:`AggregateRangeScore`; every other + numeric key becomes an :class:`AggregateScalarScore`. Non-numeric values are skipped — they are + labels or notes, not measurements. + + Names are namespaced by runner (not by agent): each run is instrumented with a single agent, so the + agent adds no disambiguation, and a reader needs to know which *backend* produced a number. + """ + if not aggregations: + return [] + multi_agent = len(aggregations) > 1 + scores: list[AggregateScore] = [] + for agent_name, payload in sorted(aggregations.items()): + agent_metrics = payload.get("agent_metrics") if isinstance(payload, Mapping) else None + if not isinstance(agent_metrics, Mapping): + continue + # One agent per run is the norm, so `runner.gym.` reads cleanly; qualify by agent only + # when a run really did produce several, where the unqualified names would collide. + prefix = f"runner.gym.{agent_name}." if multi_agent else "runner.gym." + scores.extend(_scores_from_agent_metrics(agent_metrics, prefix=prefix)) + return scores + + +def _scores_from_agent_metrics(agent_metrics: Mapping[str, Any], *, prefix: str) -> list[AggregateScore]: + families: dict[str, dict[str, float]] = {} + scalars: dict[str, float] = {} + for key, value in agent_metrics.items(): + number = _as_float(value) + if number is None: + continue + stat, _, metric = key.partition("/") + if metric and stat in _GYM_STAT_FAMILY: + families.setdefault(metric, {})[stat] = number + else: + scalars[key] = number + + scores: list[AggregateScore] = [] + for metric, stats in sorted(families.items()): + if set(stats) < set(_GYM_STAT_FAMILY): + # Not a distribution we can vouch for; keep each key as the standalone number it may well be. + scalars.update({f"{stat}/{metric}": value for stat, value in stats.items()}) + continue + if metric in _GYM_REDUNDANT_METRICS: + continue + scores.append( + AggregateRangeScore( + name=f"{prefix}{metric}", + # Gym reports the statistics but not the sample size behind them, and inventing one + # would misreport coverage. `None` says "unknown"; 0 would assert nothing was evaluated. + count=None, + nan_count=0, + mean=stats["mean"], + min=stats["min"], + max=stats["max"], + median=stats["median"], + # Gym computes this with pandas (ddof=1), so it is the sample standard deviation. + sample_std_dev=stats["std"], + sample_variance=stats["std"] ** 2, + ) + ) + scores.extend( + AggregateScalarScore(name=f"{prefix}{key}", count=None, nan_count=0, value=value) + for key, value in sorted(scalars.items()) + if key not in _GYM_REDUNDANT_METRICS + ) + return scores + + +def _as_float(value: Any) -> float | None: + """``float`` for a numeric Gym metric value; None for anything else (bools included: not measurements).""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) if math.isfinite(value) else None + + def _ensure_fresh_output(rollouts_path: Path) -> None: """Enforce one Gym run per output dir (the AgentEvaluator convention). diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py index 17faef3584..68f0f50c40 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py @@ -25,6 +25,7 @@ CandidateEvidence, EvidenceDescriptor, ) +from nemo_platform.beta.evaluator.values.results import AggregateScore from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator @@ -141,6 +142,22 @@ def callable_identity(target: object) -> str: return f"{module}.{name}" if module else name +@runtime_checkable +class RunAggregationsProvider(Protocol): + """Optional companion to :class:`AgentTaskRunner`: a runner that computed its own run-level + aggregations (a backend's pass@k, reward profile, environment-specific metrics) exposes them here, + mapped onto the SDK's typed aggregate scores. The evaluator calls this after ``run_tasks``; + implementers stash their numbers during the run and convert them here. + + Returned scores are merged into ``summary.scores``, so a backend's own figures sit alongside the + SDK's and are addressable by name the same way. Implementers must namespace names under + ``runner..`` so an imported figure is never mistaken for one the SDK computed. Runners + with no run-level aggregations simply don't implement this protocol. + """ + + def run_aggregate_scores(self) -> Sequence[AggregateScore]: ... + + @runtime_checkable class AgentTrialSerde(Protocol): """Read/write a single stored trial artifact as an :class:`AgentEvalTrial`. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py index d802bf45f7..3df0f7a395 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/metrics/aggregation.py @@ -183,7 +183,7 @@ def _compute_percentile(sorted_values: list[float], percentile: float) -> float: return sorted_values[lower_idx] + frac * (sorted_values[lower_idx + 1] - sorted_values[lower_idx]) -def _compute_percentiles(sorted_values: list[float]) -> Percentiles: +def compute_percentiles(sorted_values: list[float]) -> Percentiles: """Compute the fixed percentile set used by SDK aggregate output. Args: @@ -335,11 +335,18 @@ def aggregate_metrics( n = len(values) mean = results.stats.mean or 0 - # Use population variance because these rows are the full evaluation set, - # not a sample intended to estimate a larger population. - variance = sum((v - mean) ** 2 for v in values) / n if n > 0 else 0 + # Report both conventions under explicit names rather than leaving the divisor implicit. + # `variance`/`stddev` stay population (divide by n): these rows are the full evaluation set, + # not a sample intended to estimate a larger population. The sample (n-1) figures are also + # provided for callers estimating the spread of the process the values were drawn from, and + # are undefined for a single value. + sum_sq_dev = sum((v - mean) ** 2 for v in values) + variance = sum_sq_dev / n if n > 0 else 0 results.stats.variance = variance results.stats.stddev = math.sqrt(variance) + sample_variance = sum_sq_dev / (n - 1) if n > 1 else None + results.stats.sample_variance = sample_variance + results.stats.sample_stddev = math.sqrt(sample_variance) if sample_variance is not None else None aggregated_scores: list[AggregateScore] = [] for score_name, metric_score in aggregated_results.items(): @@ -356,6 +363,12 @@ def aggregate_metrics( base_max = stats.max if stats.max is not None else base_mean base_variance = stats.variance if stats.variance is not None else (None if base_mean is None else 0.0) base_std_dev = stats.stddev if stats.stddev is not None else (None if base_mean is None else 0.0) + # Sample stats stay None when undefined (fewer than two values) rather than defaulting to 0.0. + base_sample_variance = stats.sample_variance + base_sample_std_dev = stats.sample_stddev + # Derived from the same helper that produces p50, so `median` and `percentiles.p50` agree + # exactly wherever both are present (rubric scores carry no percentiles but still get a median). + base_median = _compute_percentile(sorted(values), 50) if values else None if base_count == 0: base_sum = None @@ -364,6 +377,9 @@ def aggregate_metrics( base_max = None base_variance = None base_std_dev = None + base_sample_variance = None + base_sample_std_dev = None + base_median = None if has_rubric.get(score_name): rubric_dist = [ @@ -386,8 +402,11 @@ def aggregate_metrics( mean=base_mean, min=base_min, max=base_max, + median=base_median, variance=base_variance, std_dev=base_std_dev, + sample_variance=base_sample_variance, + sample_std_dev=base_sample_std_dev, rubric_distribution=rubric_dist, mode_category=mode_category, ) @@ -396,7 +415,7 @@ def aggregate_metrics( if values: # Range scores get richer distribution metadata than rubric scores. sorted_values = sorted(values) - percentiles = _compute_percentiles(sorted_values) + percentiles = compute_percentiles(sorted_values) histogram = _compute_histogram(values) else: percentiles = None @@ -411,8 +430,11 @@ def aggregate_metrics( mean=base_mean, min=base_min, max=base_max, + median=base_median, variance=base_variance, std_dev=base_std_dev, + sample_variance=base_sample_variance, + sample_std_dev=base_sample_std_dev, percentiles=percentiles, histogram=histogram, ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py index dc1f8220d3..462d0e0ace 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/values/results.py @@ -22,8 +22,11 @@ "mean", "min", "max", + "median", "std_dev", "variance", + "sample_std_dev", + "sample_variance", # Range-specific fields "score_type", "percentiles", @@ -180,6 +183,15 @@ class ScoreStats(BaseModel): default=None, description="""The population standard deviation, (note: not the sample standard deviation).""", ) + sample_variance: float | None = Field( + default=None, + description="The sample (Bessel-corrected, n-1) variance. None when fewer than two values.", + ) + sample_stddev: float | None = Field( + default=None, + description="The sample (Bessel-corrected, n-1) standard deviation, estimating the spread of the " + "process the values were drawn from. None when fewer than two values (undefined, not zero).", + ) stderr: float | None = Field(default=None, description="The standard error.") nan_count: int | None = Field( default=None, @@ -189,7 +201,9 @@ class ScoreStats(BaseModel): default=None, description="The distribution of the rubric grading criteria for the score." ) - @field_serializer("sum", "sum_squared", "min", "max", "mean", "variance", "stddev", "stderr") + @field_serializer( + "sum", "sum_squared", "min", "max", "mean", "variance", "stddev", "sample_variance", "sample_stddev", "stderr" + ) def serialize_nan(self, v: float | None) -> float | str | None: """Serialize NaN stats as string values for JSON compatibility. @@ -279,14 +293,42 @@ class AggregateScoreBase(BaseModel): model_config = ConfigDict(extra="forbid") name: str = Field(description="Name of the score.") - count: int = Field(description="Number of samples evaluated (excluding NaN).") + count: int | None = Field( + default=None, + description="Number of samples evaluated (excluding NaN). None when the sample size is unknown " + "— e.g. a figure imported from a backend that reports statistics without the n behind them. " + "Distinct from 0, which asserts that nothing was evaluated.", + ) nan_count: int = Field(description="Number of samples that produced NaN scores.") sum: float | None = Field(default=None, description="Sum of all score values.") mean: float | None = Field(default=None, description="Mean score value.") min: float | None = Field(default=None, description="Minimum score value.") max: float | None = Field(default=None, description="Maximum score value.") - std_dev: float | None = Field(default=None, description="Standard deviation of the scores.") - variance: float | None = Field(default=None, description="Variance of the scores.") + median: float | None = Field( + default=None, + description="Median score value. Equal to percentiles.p50 when a percentile distribution is " + "also present; carried separately because a backend may report a median without one.", + ) + std_dev: float | None = Field( + default=None, + description="Population standard deviation of the scores (divides by n). Describes the spread of " + "the values actually evaluated. See sample_std_dev to estimate the spread of the wider process.", + ) + variance: float | None = Field( + default=None, + description="Population variance of the scores (divides by n). See sample_variance.", + ) + sample_std_dev: float | None = Field( + default=None, + description="Sample standard deviation of the scores (Bessel-corrected, divides by n-1). Estimates " + "the spread of the process the values were drawn from — the right choice when repeated trials " + "sample a stochastic system. None when fewer than two values (undefined, not zero).", + ) + sample_variance: float | None = Field( + default=None, + description="Sample variance of the scores (Bessel-corrected, divides by n-1). None when fewer " + "than two values.", + ) class AggregateRangeScore(AggregateScoreBase): @@ -339,7 +381,38 @@ def _serialize(self, handler): return data -AggregateScore = AggregateRangeScore | AggregateRubricScore +class AggregateScalarScore(AggregateScoreBase): + """A single pre-computed value with no underlying distribution available. + + For figures a backend reports as one number (e.g. an environment's own ``pass@1`` or Elo) rather + than a set of per-sample values the SDK could aggregate itself. ``value`` carries the number; + ``mean``/``min``/``max`` are left unset because there is no sample to describe. Distinct from + :class:`AggregateRangeScore` so a reader can tell "this is the whole story" from "this summarizes + ``count`` samples", instead of seeing a range score with a suspicious ``count`` of 1. + """ + + score_type: Literal["scalar"] = Field(default="scalar", description="Type of score.") + value: float = Field(description="The reported value.") + + _include_fields: frozenset[str] | None = None + + def with_fields(self, fields: frozenset[AggregateFieldName]) -> Self: + """Return a copy configured to serialize only the specified fields.""" + copy = self.model_copy() + object.__setattr__(copy, "_include_fields", {*fields, "name", "count"}) + return copy + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self._include_fields is not None: + # Always include required fields (name, count, value), plus requested fields + fields_to_include = self._include_fields | {"name", "count", "value"} + return {k: v for k, v in data.items() if k in fields_to_include} + return data + + +AggregateScore = AggregateRangeScore | AggregateRubricScore | AggregateScalarScore class AggregatedMetricResult(BaseModel):