feat(evaluator)!: typed run metadata and a required runner identity contract

packages/nemo_evaluator_sdk/examples/codex_docker/example.py

@@ -134,7 +134,7 @@ async def evaluate(
             output_dir=resolved_output_dir,
             parallelism=1,
             write_dashboard=write_dashboard,
-            benchmark={"name": "codex-docker-evidence-sanity"},
+            labels={"scenario": "codex-docker-evidence-sanity"},
         ),
     )
 

packages/nemo_evaluator_sdk/examples/profbench/README.md

@@ -134,7 +134,7 @@ async def main() -> None:
             output_dir=output_dir,
             run_id="profbench-code-sandbox-smoke",
             parallelism=1,
-            benchmark={**benchmark.metadata, "score_source": "docker_sandbox_and_live_judge"},
+            labels={**{k: str(v) for k, v in benchmark.metadata.items()}, "score_source": "docker_sandbox_and_live_judge"},
             write_dashboard=False,
         ),
     )
@@ -260,7 +260,7 @@ result = await AgentEvaluator().run(
     config=AgentEvalRunConfig(
         output_dir=output_dir,
         params=params,
-        benchmark={**benchmark.metadata, "score_source": "fresh_candidate_and_live_judge"},
+        labels={**{k: str(v) for k, v in benchmark.metadata.items()}, "score_source": "fresh_candidate_and_live_judge"},
         ...
     ),
 )
@@ -511,6 +511,6 @@ What happens in the full live-candidate path:
 4. `load_profbench()` loads tasks with `include_cached_fulfilments=False`, so cached labels are removed and the metric must call the judge.
 5. `AgentEvaluator.run(tasks=..., target=evaluated_model, ...)` generates fresh candidate trials: for each task it calls `_generate_sample()` against the evaluated model and converts the returned sample into an `AgentEvalTrial`.
 6. The evaluator then scores those generated trials with `ProfBenchRubricMetric`. For each rubric criterion it calls `ProfBenchModelJudge`, which calls `_generate_sample()` against the judge model, parses the judge output into a yes/no decision, writes `evidence/judge-*.json`, and returns `MetricResult` outputs.
-7. `AgentEvaluator` builds the summary, persists the run bundle (`benchmark.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`.
+7. `AgentEvaluator` builds the summary, persists the run bundle (`metadata.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`.
 
 The evaluated model produces the candidate answer and the judge model evaluates each rubric criterion. They can point to the same model configuration, but the code treats them as separate roles.

packages/nemo_evaluator_sdk/examples/profbench/runner.py

@@ -100,7 +100,7 @@ async def run_profbench_mode(
     target: AgentEvalTarget | None = None
     trials: list[AgentEvalTrial] | None = None
     params: RunConfigOnlineModel | None = None
-    benchmark_meta = dict(benchmark.metadata)
+    benchmark_labels = {key: str(value) for key, value in benchmark.metadata.items()}
     if mode is ProfBenchMode.LIVE_CANDIDATE:
         target, params, score_source, effective_codex_runtime = _live_candidate_target(
             agent=agent,
@@ -110,11 +110,11 @@ async def run_profbench_mode(
         )
         if effective_codex_runtime is not None:
             print(f"Codex runtime: {effective_codex_runtime}")
-        benchmark_meta["score_source"] = score_source
+        benchmark_labels["score_source"] = score_source
     else:
         trials = benchmark.trials
         if mode is ProfBenchMode.LIVE_JUDGE:
-            benchmark_meta["score_source"] = "live_judge"
+            benchmark_labels["score_source"] = "live_judge"
 
     result = await AgentEvaluator().run(
         tasks=benchmark.tasks,
@@ -124,7 +124,7 @@ async def run_profbench_mode(
             output_dir=output_dir,
             run_id=f"{run_instance_id}-{mode.value}",
             params=params,
-            benchmark=benchmark_meta,
+            labels=benchmark_labels,
             write_dashboard=False,
         ),
     )

packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py

@@ -22,7 +22,7 @@
 import yaml
 from nemo_evaluator_sdk.agent_eval.runtimes.environment import AgentEnvironmentProvider, EnvRunSpec
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
-from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial
+from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo
 
 from .platform_runtime import (
     DEFAULT_LOCAL_NMP_BASE_URL,
@@ -189,6 +189,26 @@ def __init__(self, config: AutConfig, *, environment: AgentEnvironmentProvider |
         self.config = config
         self.environment = environment or PlatformDockerEnvironmentProvider()
 
+    def runner_info(self) -> RunnerInfo:
+        """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract).
+
+        Enumerates fields rather than dumping ``AutConfig``: it also carries ``nvidia_api_key``,
+        ``inference_nvidia_api_key`` and ``anthropic_api_key``, and this is persisted with the run.
+        """
+        return RunnerInfo(
+            name="example_nat_aut",
+            kind="runner",
+            config={
+                "aut_agent_name": self.config.aut_agent_name,
+                "aut_agent_config": str(self.config.aut_agent_config) if self.config.aut_agent_config else None,
+                "aut_seed_providers": self.config.aut_seed_providers,
+                "aut_health_wait_seconds": self.config.aut_health_wait_seconds,
+                "agent_model": self.config.agent_model,
+                "nmp_base_url": self.config.nmp_base_url,
+                "timeout_sec": self.config.timeout_sec,
+            },
+        )
+
     async def run_tasks(
         self,
         tasks: Sequence[AgentEvalTask],

packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py

@@ -143,7 +143,7 @@ def summarize_run(
     total = len(task_ids)
     return {
         "run_id": result.run_id,
-        "benchmark": result.benchmark,
+        "labels": result.metadata.labels,
         "total_tasks": total,
         "passed_tasks": passed,
         "pass_rate": (passed / total) if total else 0.0,

packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py

@@ -58,7 +58,7 @@ async def run_tasks(
         tasks: Sequence[AgentEvalTask],
         *,
         target: AgentEvalTarget,
-        benchmark: dict[str, object] | None = None,
+        labels: dict[str, str] | None = None,
         output_dir: Path | None = None,
         run_id: str | None = None,
         prepare_task: Callable[[AgentEvalTask], None] | None = None,
@@ -72,7 +72,7 @@ async def run_tasks(
         result = await AgentEvaluator().run(
             tasks=prepared,
             target=target,
-            config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
+            config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels),
         )
         self._maybe_write_gate(result)
         return result
@@ -82,7 +82,7 @@ async def score_trials(
         tasks: Sequence[AgentEvalTask],
         *,
         trials: Sequence[AgentEvalTrial],
-        benchmark: dict[str, object] | None = None,
+        labels: dict[str, str] | None = None,
         output_dir: Path | None = None,
         run_id: str | None = None,
     ) -> AgentEvalResult:
@@ -91,7 +91,7 @@ async def score_trials(
         result = await AgentEvaluator().run(
             tasks=prepared,
             trials=list(trials),
-            config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
+            config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels),
         )
         self._maybe_write_gate(result)
         return result
@@ -101,14 +101,14 @@ def _run_config(
         *,
         output_dir: Path | None,
         run_id: str | None,
-        benchmark: dict[str, object] | None,
+        labels: dict[str, str] | None,
     ) -> AgentEvalRunConfig:
         return AgentEvalRunConfig(
             output_dir=output_dir,
             run_id=run_id,
             parallelism=self.config.parallelism,
             write_dashboard=self.config.write_dashboard,
-            benchmark=dict(benchmark or {}),
+            labels=dict(labels or {}),
         )
 
     def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask:

packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py

@@ -37,6 +37,7 @@
     AgentEvalTrial,
     AgentEvalTrialStatus,
     AgentOutput,
+    RunnerInfo,
     resolve_trial_status,
     standard_evidence_descriptors,
 )
@@ -551,6 +552,24 @@ def __init__(
         self.config = config or NatWorkflowConfig()
         self.environment = environment or PlatformDockerEnvironmentProvider()
 
+    def runner_info(self) -> RunnerInfo:
+        """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract).
+
+        Enumerates fields rather than dumping ``NatWorkflowConfig``: it also carries ``nvidia_api_key``,
+        and this is persisted with the run.
+        """
+        return RunnerInfo(
+            name="example_nat_workflow",
+            kind="runner",
+            config={
+                "nmp_base_url": self.config.nmp_base_url,
+                "agent_model": self.config.agent_model,
+                "timeout_sec": self.config.timeout_sec,
+                "run_verify": self.config.run_verify,
+                "docker_extra_args": list(self.config.docker_extra_args),
+            },
+        )
+
     async def run_tasks(
         self,
         tasks: Sequence[AgentEvalTask],

packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py

@@ -77,7 +77,7 @@ async def run_online(task_names: list[str], *, output_dir: Path, min_pass_rate:
     return await _pipeline(min_pass_rate).run_tasks(
         tasks,
         target=runtime,
-        benchmark={"benchmark": "run-agent-eval", "mode": "online"},
+        labels={"example": "run-agent-eval", "mode": "online"},
         output_dir=output_dir,
     )
 
@@ -132,7 +132,7 @@ async def run_agentic_task(
     return await _pipeline(min_pass_rate, extra_metrics=extra_metrics).run_tasks(
         [task],
         target=runtime,
-        benchmark={"benchmark": "agentic-use", "task": task_name, "backend": backend},
+        labels={"example": "agentic-use", "task": task_name, "backend": backend},
         output_dir=output_dir,
         prepare_task=lambda t: ensure_task_image(t, skip_build=skip_build),
     )
@@ -145,7 +145,7 @@ async def rescore(rescore_dirs: list[Path], *, output_dir: Path, min_pass_rate:
     return await _pipeline(min_pass_rate).score_trials(
         tasks,
         trials=trials,
-        benchmark={"benchmark": "run-agent-eval", "mode": "offline"},
+        labels={"example": "run-agent-eval", "mode": "offline"},
         output_dir=output_dir,
     )
 

packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py

@@ -30,6 +30,7 @@
     AgentEvalTrial,
     AgentEvalTrialStatus,
     AgentOutput,
+    RunnerInfo,
     resolve_trial_status,
     standard_evidence_descriptors,
 )
@@ -77,6 +78,18 @@ class WorkflowAgentRuntime:
     def __init__(self, config: WorkflowRuntimeConfig | None = None) -> None:
         self.config = config or WorkflowRuntimeConfig()
 
+    def runner_info(self) -> RunnerInfo:
+        """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract)."""
+        return RunnerInfo(
+            name="example_workflow",
+            kind="runner",
+            config={
+                "command": list(self.config.command),
+                "timeout_s": self.config.timeout_s,
+                "agent_model": self.config.agent_model,
+            },
+        )
+
     async def run_tasks(
         self,
         tasks: Sequence[AgentEvalTask],

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -12,6 +12,8 @@
 from collections import defaultdict
 from collections.abc import Awaitable, Callable, Sequence
 from datetime import UTC, datetime
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as package_version
 from logging import getLogger
 from pathlib import Path
 from typing import Any, cast, overload
@@ -21,7 +23,7 @@
 import nemo_evaluator_sdk.inference as inference
 from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard
 from nemo_evaluator_sdk.agent_eval.persistence import persist_run
-from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary, RunMetadata
 from nemo_evaluator_sdk.agent_eval.scores import (
     AgentEvalDiagnostic,
     AgentEvalDiagnosticSeverity,
@@ -35,6 +37,7 @@
     AgentEvalTrialStatus,
     AgentOutput,
     AgentTaskRunner,
+    RunnerInfo,
 )
 from nemo_evaluator_sdk.agent_inference import (
     AgentInferenceContext,
@@ -155,6 +158,7 @@ async def run(
 
         run_id = resolved_config.run_id or _new_run_id()
         runtime_config = resolved_config.model_copy(update={"run_id": run_id})
+        started_at = datetime.now(UTC)
 
         # Branch on which seam was supplied so the type checker can narrow ``target`` to a
         # concrete ``AgentEvalTarget`` without a cast.
@@ -172,14 +176,22 @@ async def run(
             config=runtime_config,
             run_id=run_id,
         )
-        benchmark = {**_benchmark_metadata(task_list), **runtime_config.benchmark}
+        finished_at = datetime.now(UTC)
+        metadata = RunMetadata(
+            labels=dict(runtime_config.labels),
+            target=_describe_target(target, runtime_config.params),
+            started_at=started_at,
+            finished_at=finished_at,
+            duration_sec=(finished_at - started_at).total_seconds(),
+            sdk_version=_sdk_version(),
+        )
         result = AgentEvalResult(
             run_id=run_id,
             tasks=task_list,
             trials=trial_list,
             scores=scores,
             summary=AgentEvalSummary.from_scores(scores, tasks=task_list),
-            benchmark=benchmark,
+            metadata=metadata,
         )
 
         if runtime_config.output_dir is not None:
@@ -686,11 +698,38 @@ def _is_completions_endpoint(url: str) -> bool:
     return path.endswith("/completions") and not path.endswith("/chat/completions")
 
 
-def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]:
-    benchmarks = sorted({str(task.metadata.get("benchmark")) for task in tasks if task.metadata.get("benchmark")})
-    if not benchmarks:
-        return {}
-    return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks}
+def _sdk_version() -> str | None:
+    try:
+        return package_version("nemo-evaluator-sdk")
+    except PackageNotFoundError:  # pragma: no cover - only when running from an uninstalled tree
+        return None
+
+
+def _describe_target(
+    target: AgentEvalTarget | None,
+    params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None,
+) -> RunnerInfo:
+    """Identify what produced the trials, for the run's provenance.
+
+    Runners identify themselves via the required :meth:`AgentTaskRunner.runner_info`; trials supplied
+    directly have no runner.
+
+    Models and agents are described by name *and* the settings they were invoked with — the endpoint
+    ``url``, plus the whole ``params`` object (temperature, max_tokens, reasoning effort, system prompt,
+    retries, ...). A name alone is not an identity: the same model name served from two different URLs,
+    or at two different temperatures, would otherwise record identical provenance. ``params`` is dumped
+    whole rather than cherry-picked, because a filtered subset is what bites you later when the omitted
+    field turns out to be the one that mattered. It carries no credentials — ``Model.api_key_secret`` is
+    a reference on the model, and ``default_headers`` is excluded from serialization.
+    """
+    if target is None:
+        return RunnerInfo(name="imported", kind="imported")
+    if isinstance(target, (Model, AgentBase)):
+        config: dict[str, Any] = {"url": getattr(target, "url", None)}
+        if params is not None:
+            config["params"] = params.model_dump(mode="json", exclude_none=True)
+        return RunnerInfo(name=target.name, kind="model" if isinstance(target, Model) else "agent", config=config)
+    return target.runner_info()
 
 
 def _persist_with_optional_dashboard(

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py

@@ -20,7 +20,7 @@ def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalRes
     path = Path(output_dir)
     path.mkdir(parents=True, exist_ok=True)
 
-    _write_json(path / "benchmark.json", result.benchmark)
+    _write_json(path / "metadata.json", result.metadata)
     _write_jsonl(path / "tasks.jsonl", result.tasks)
     _write_trials(path / "trials.jsonl", result.trials, base=path)
     _write_jsonl(path / "scores.jsonl", result.scores)
@@ -37,7 +37,7 @@ def _run_manifest(result: AgentEvalResult) -> dict[str, Any]:
         "output_dir": str(result.output_dir) if result.output_dir is not None else None,
         "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None,
         "artifacts": {
-            "benchmark": "benchmark.json",
+            "metadata": "metadata.json",
             "tasks": "tasks.jsonl",
             "trials": "trials.jsonl",
             "scores": "scores.jsonl",