From 45fb359601250988faad1a99b47d71d628726b1b Mon Sep 17 00:00:00 2001 From: Sandy Chapman Date: Tue, 4 Aug 2026 09:27:11 -0300 Subject: [PATCH] feat(evaluator)!: typed run metadata and a required runner identity contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the untyped `benchmark: dict[str, Any]` bag on AgentEvalResult with typed run provenance, and require runners to identify themselves. A finished run could not answer "what produced this, with what settings, and how long did it take?" — there was no timing, no target identity, and no SDK version. Callers improvised provenance inside the benchmark dict with no convention ({"name": ...} vs {"benchmark": ...}, plus mode/backend/task/ score_source), and the auto-derived value shape-shifted between str and list[str] depending on how many benchmarks the tasks declared. - AgentEvalResult.benchmark -> metadata: RunMetadata, carrying typed labels, target, started_at/finished_at/duration_sec, and sdk_version. - runner_info() -> RunnerInfo is now part of the AgentTaskRunner protocol. Identity is universal rather than an optional capability, so every shipped runner implements it with a curated name (gym, harbor, docker_sandbox, callable, codex_cli, codex_docker_cli, fabric, fabric_container) plus the settings that shape its results. Credentials are deliberately excluded. - Drop `benchmark` entirely: nothing computed on it, it duplicated what labels already express, and it invited misuse as a run name. - Bundle artifact benchmark.json -> metadata.json. Provenance that omits a result-shaping setting is worse than none, because two runs that behaved differently would record identical metadata. So each runner surfaces its own knobs: gym its bind_resources_server/env_overrides/reward_key; harbor its n_attempts and configured job location; docker_sandbox its effective instructions; codex its codex_bin and prompt builder; fabric its adapter, skills, effective model, and capture_trajectory. Fabric's model is resolved the way _compose_config resolves it — an explicit argument wins, otherwise the config's default — since a model supplied only through config is what actually runs. capture_trajectory is recorded because with it off no relay/ATIF exporter runs and the trial carries no trajectory evidence. Two identity values were unstable across identical runs and are now derived from declared names instead: FabricContainerRuntime recorded str(provider), a memory-address repr, and callable identities used a bare __qualname__, which is ambiguous across modules. The evaluator plugin forwarded AgentEvalSpec.benchmark into a parameter that no longer exists; the spec field is renamed to labels to match. AgentEvalSpec is plugin-internal and absent from the OpenAPI surface, so this is not an API-visible change. Harbor's concrete job directory is deliberately not resolved early: the native job name defaults to a run-time timestamp, so the configured jobs_dir/job_name are recorded rather than a fabricated path. Tests cover all eight shipped runners, asserting both the curated name and the presence of each runner's result-shaping configuration. BREAKING CHANGE: AgentTaskRunner now requires runner_info(). Because the protocol is runtime_checkable and isinstance-dispatched, a runner without it no longer matches and raises NotImplementedError: unsupported agent-eval target type. Signed-off-by: Sandy Chapman --- .../examples/codex_docker/example.py | 2 +- .../examples/profbench/README.md | 6 +- .../examples/profbench/runner.py | 8 +- .../examples/run_agent_eval/aut_runtime.py | 22 +- .../examples/run_agent_eval/gating.py | 2 +- .../examples/run_agent_eval/pipeline.py | 12 +- .../run_agent_eval/platform_runtime.py | 19 ++ .../examples/run_agent_eval/run_agent_eval.py | 6 +- .../run_agent_eval/workflow_runtime.py | 13 + .../agent_eval/evaluator.py | 55 +++- .../agent_eval/persistence.py | 4 +- .../nemo_evaluator_sdk/agent_eval/results.py | 34 +- .../agent_eval/runtimes/callable_runtime.py | 19 +- .../agent_eval/runtimes/codex/runtime.py | 29 +- .../agent_eval/runtimes/docker_sandbox.py | 15 +- .../runtimes/fabric/container_runtime.py | 18 +- .../agent_eval/runtimes/fabric/runtime.py | 39 ++- .../agent_eval/runtimes/gym_runtime.py | 54 +++- .../agent_eval/runtimes/harbor_runtime.py | 39 +++ .../nemo_evaluator_sdk/agent_eval/tasks.py | 6 +- .../nemo_evaluator_sdk/agent_eval/trials.py | 39 +++ .../agent_eval/test_codex_docker_example.py | 5 +- .../tests/agent_eval/test_evaluator.py | 9 +- .../tests/agent_eval/test_persistence.py | 2 - .../tests/agent_eval/test_run_metadata.py | 303 ++++++++++++++++++ .../src/nemo_evaluator/jobs/agent_evaluate.py | 4 +- .../src/nemo_evaluator/jobs/agent_spec.py | 5 +- .../beta/evaluator/agent_eval/evaluator.py | 55 +++- .../beta/evaluator/agent_eval/persistence.py | 4 +- .../beta/evaluator/agent_eval/results.py | 34 +- .../agent_eval/runtimes/callable_runtime.py | 19 +- .../agent_eval/runtimes/codex/runtime.py | 29 +- .../agent_eval/runtimes/docker_sandbox.py | 15 +- .../runtimes/fabric/container_runtime.py | 18 +- .../agent_eval/runtimes/fabric/runtime.py | 39 ++- .../agent_eval/runtimes/gym_runtime.py | 54 +++- .../agent_eval/runtimes/harbor_runtime.py | 39 +++ .../beta/evaluator/agent_eval/tasks.py | 6 +- .../beta/evaluator/agent_eval/trials.py | 39 +++ 39 files changed, 1046 insertions(+), 74 deletions(-) create mode 100644 packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py diff --git a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py b/packages/nemo_evaluator_sdk/examples/codex_docker/example.py index 589fbaf7d2..49c46c9d68 100644 --- a/packages/nemo_evaluator_sdk/examples/codex_docker/example.py +++ b/packages/nemo_evaluator_sdk/examples/codex_docker/example.py @@ -134,7 +134,7 @@ async def evaluate( output_dir=resolved_output_dir, parallelism=1, write_dashboard=write_dashboard, - benchmark={"name": "codex-docker-evidence-sanity"}, + labels={"scenario": "codex-docker-evidence-sanity"}, ), ) diff --git a/packages/nemo_evaluator_sdk/examples/profbench/README.md b/packages/nemo_evaluator_sdk/examples/profbench/README.md index 7780696c54..cc622daae2 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/README.md +++ b/packages/nemo_evaluator_sdk/examples/profbench/README.md @@ -134,7 +134,7 @@ async def main() -> None: output_dir=output_dir, run_id="profbench-code-sandbox-smoke", parallelism=1, - benchmark={**benchmark.metadata, "score_source": "docker_sandbox_and_live_judge"}, + labels={**{k: str(v) for k, v in benchmark.metadata.items()}, "score_source": "docker_sandbox_and_live_judge"}, write_dashboard=False, ), ) @@ -260,7 +260,7 @@ result = await AgentEvaluator().run( config=AgentEvalRunConfig( output_dir=output_dir, params=params, - benchmark={**benchmark.metadata, "score_source": "fresh_candidate_and_live_judge"}, + labels={**{k: str(v) for k, v in benchmark.metadata.items()}, "score_source": "fresh_candidate_and_live_judge"}, ... ), ) @@ -511,6 +511,6 @@ What happens in the full live-candidate path: 4. `load_profbench()` loads tasks with `include_cached_fulfilments=False`, so cached labels are removed and the metric must call the judge. 5. `AgentEvaluator.run(tasks=..., target=evaluated_model, ...)` generates fresh candidate trials: for each task it calls `_generate_sample()` against the evaluated model and converts the returned sample into an `AgentEvalTrial`. 6. The evaluator then scores those generated trials with `ProfBenchRubricMetric`. For each rubric criterion it calls `ProfBenchModelJudge`, which calls `_generate_sample()` against the judge model, parses the judge output into a yes/no decision, writes `evidence/judge-*.json`, and returns `MetricResult` outputs. -7. `AgentEvaluator` builds the summary, persists the run bundle (`benchmark.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`. +7. `AgentEvaluator` builds the summary, persists the run bundle (`metadata.json`, `tasks.jsonl`, `trials.jsonl`, `scores.jsonl`, `summary.json`, `run.json`), and `write_example_dashboards()` writes `sdk-report.html` and the ProfBench-specific `report.html`. The evaluated model produces the candidate answer and the judge model evaluates each rubric criterion. They can point to the same model configuration, but the code treats them as separate roles. diff --git a/packages/nemo_evaluator_sdk/examples/profbench/runner.py b/packages/nemo_evaluator_sdk/examples/profbench/runner.py index d8435f32f7..7c3146d308 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/runner.py +++ b/packages/nemo_evaluator_sdk/examples/profbench/runner.py @@ -100,7 +100,7 @@ async def run_profbench_mode( target: AgentEvalTarget | None = None trials: list[AgentEvalTrial] | None = None params: RunConfigOnlineModel | None = None - benchmark_meta = dict(benchmark.metadata) + benchmark_labels = {key: str(value) for key, value in benchmark.metadata.items()} if mode is ProfBenchMode.LIVE_CANDIDATE: target, params, score_source, effective_codex_runtime = _live_candidate_target( agent=agent, @@ -110,11 +110,11 @@ async def run_profbench_mode( ) if effective_codex_runtime is not None: print(f"Codex runtime: {effective_codex_runtime}") - benchmark_meta["score_source"] = score_source + benchmark_labels["score_source"] = score_source else: trials = benchmark.trials if mode is ProfBenchMode.LIVE_JUDGE: - benchmark_meta["score_source"] = "live_judge" + benchmark_labels["score_source"] = "live_judge" result = await AgentEvaluator().run( tasks=benchmark.tasks, @@ -124,7 +124,7 @@ async def run_profbench_mode( output_dir=output_dir, run_id=f"{run_instance_id}-{mode.value}", params=params, - benchmark=benchmark_meta, + labels=benchmark_labels, write_dashboard=False, ), ) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py index 36869a9f11..57311e90bf 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/aut_runtime.py @@ -22,7 +22,7 @@ import yaml from nemo_evaluator_sdk.agent_eval.runtimes.environment import AgentEnvironmentProvider, EnvRunSpec from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo from .platform_runtime import ( DEFAULT_LOCAL_NMP_BASE_URL, @@ -189,6 +189,26 @@ def __init__(self, config: AutConfig, *, environment: AgentEnvironmentProvider | self.config = config self.environment = environment or PlatformDockerEnvironmentProvider() + def runner_info(self) -> RunnerInfo: + """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract). + + Enumerates fields rather than dumping ``AutConfig``: it also carries ``nvidia_api_key``, + ``inference_nvidia_api_key`` and ``anthropic_api_key``, and this is persisted with the run. + """ + return RunnerInfo( + name="example_nat_aut", + kind="runner", + config={ + "aut_agent_name": self.config.aut_agent_name, + "aut_agent_config": str(self.config.aut_agent_config) if self.config.aut_agent_config else None, + "aut_seed_providers": self.config.aut_seed_providers, + "aut_health_wait_seconds": self.config.aut_health_wait_seconds, + "agent_model": self.config.agent_model, + "nmp_base_url": self.config.nmp_base_url, + "timeout_sec": self.config.timeout_sec, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py index 557c2e5a06..8c5320979b 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/gating.py @@ -143,7 +143,7 @@ def summarize_run( total = len(task_ids) return { "run_id": result.run_id, - "benchmark": result.benchmark, + "labels": result.metadata.labels, "total_tasks": total, "passed_tasks": passed, "pass_rate": (passed / total) if total else 0.0, diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py index 67111f285b..d45e3f78c7 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py @@ -58,7 +58,7 @@ async def run_tasks( tasks: Sequence[AgentEvalTask], *, target: AgentEvalTarget, - benchmark: dict[str, object] | None = None, + labels: dict[str, str] | None = None, output_dir: Path | None = None, run_id: str | None = None, prepare_task: Callable[[AgentEvalTask], None] | None = None, @@ -72,7 +72,7 @@ async def run_tasks( result = await AgentEvaluator().run( tasks=prepared, target=target, - config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), + config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels), ) self._maybe_write_gate(result) return result @@ -82,7 +82,7 @@ async def score_trials( tasks: Sequence[AgentEvalTask], *, trials: Sequence[AgentEvalTrial], - benchmark: dict[str, object] | None = None, + labels: dict[str, str] | None = None, output_dir: Path | None = None, run_id: str | None = None, ) -> AgentEvalResult: @@ -91,7 +91,7 @@ async def score_trials( result = await AgentEvaluator().run( tasks=prepared, trials=list(trials), - config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark), + config=self._run_config(output_dir=output_dir, run_id=run_id, labels=labels), ) self._maybe_write_gate(result) return result @@ -101,14 +101,14 @@ def _run_config( *, output_dir: Path | None, run_id: str | None, - benchmark: dict[str, object] | None, + labels: dict[str, str] | None, ) -> AgentEvalRunConfig: return AgentEvalRunConfig( output_dir=output_dir, run_id=run_id, parallelism=self.config.parallelism, write_dashboard=self.config.write_dashboard, - benchmark=dict(benchmark or {}), + labels=dict(labels or {}), ) def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask: diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py index 8c97094d75..f8dc743ca8 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py @@ -37,6 +37,7 @@ AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, + RunnerInfo, resolve_trial_status, standard_evidence_descriptors, ) @@ -551,6 +552,24 @@ def __init__( self.config = config or NatWorkflowConfig() self.environment = environment or PlatformDockerEnvironmentProvider() + def runner_info(self) -> RunnerInfo: + """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract). + + Enumerates fields rather than dumping ``NatWorkflowConfig``: it also carries ``nvidia_api_key``, + and this is persisted with the run. + """ + return RunnerInfo( + name="example_nat_workflow", + kind="runner", + config={ + "nmp_base_url": self.config.nmp_base_url, + "agent_model": self.config.agent_model, + "timeout_sec": self.config.timeout_sec, + "run_verify": self.config.run_verify, + "docker_extra_args": list(self.config.docker_extra_args), + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py index 3d564eae6e..60239e9cf1 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py @@ -77,7 +77,7 @@ async def run_online(task_names: list[str], *, output_dir: Path, min_pass_rate: return await _pipeline(min_pass_rate).run_tasks( tasks, target=runtime, - benchmark={"benchmark": "run-agent-eval", "mode": "online"}, + labels={"example": "run-agent-eval", "mode": "online"}, output_dir=output_dir, ) @@ -132,7 +132,7 @@ async def run_agentic_task( return await _pipeline(min_pass_rate, extra_metrics=extra_metrics).run_tasks( [task], target=runtime, - benchmark={"benchmark": "agentic-use", "task": task_name, "backend": backend}, + labels={"example": "agentic-use", "task": task_name, "backend": backend}, output_dir=output_dir, prepare_task=lambda t: ensure_task_image(t, skip_build=skip_build), ) @@ -145,7 +145,7 @@ async def rescore(rescore_dirs: list[Path], *, output_dir: Path, min_pass_rate: return await _pipeline(min_pass_rate).score_trials( tasks, trials=trials, - benchmark={"benchmark": "run-agent-eval", "mode": "offline"}, + labels={"example": "run-agent-eval", "mode": "offline"}, output_dir=output_dir, ) diff --git a/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py index 0906e4afdc..d9284ff47c 100644 --- a/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py +++ b/packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py @@ -30,6 +30,7 @@ AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, + RunnerInfo, resolve_trial_status, standard_evidence_descriptors, ) @@ -77,6 +78,18 @@ class WorkflowAgentRuntime: def __init__(self, config: WorkflowRuntimeConfig | None = None) -> None: self.config = config or WorkflowRuntimeConfig() + def runner_info(self) -> RunnerInfo: + """Identify this runtime and the settings that shape its results (the AgentTaskRunner contract).""" + return RunnerInfo( + name="example_workflow", + kind="runner", + config={ + "command": list(self.config.command), + "timeout_s": self.config.timeout_s, + "agent_model": self.config.agent_model, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index 2880072880..c42b240bfe 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -12,6 +12,8 @@ from collections import defaultdict from collections.abc import Awaitable, Callable, Sequence from datetime import UTC, datetime +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as package_version from logging import getLogger from pathlib import Path from typing import Any, cast, overload @@ -21,7 +23,7 @@ import nemo_evaluator_sdk.inference as inference from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard from nemo_evaluator_sdk.agent_eval.persistence import persist_run -from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary, RunMetadata from nemo_evaluator_sdk.agent_eval.scores import ( AgentEvalDiagnostic, AgentEvalDiagnosticSeverity, @@ -35,6 +37,7 @@ AgentEvalTrialStatus, AgentOutput, AgentTaskRunner, + RunnerInfo, ) from nemo_evaluator_sdk.agent_inference import ( AgentInferenceContext, @@ -155,6 +158,7 @@ async def run( run_id = resolved_config.run_id or _new_run_id() runtime_config = resolved_config.model_copy(update={"run_id": run_id}) + started_at = datetime.now(UTC) # Branch on which seam was supplied so the type checker can narrow ``target`` to a # concrete ``AgentEvalTarget`` without a cast. @@ -172,14 +176,22 @@ async def run( config=runtime_config, run_id=run_id, ) - benchmark = {**_benchmark_metadata(task_list), **runtime_config.benchmark} + finished_at = datetime.now(UTC) + metadata = RunMetadata( + labels=dict(runtime_config.labels), + target=_describe_target(target, runtime_config.params), + started_at=started_at, + finished_at=finished_at, + duration_sec=(finished_at - started_at).total_seconds(), + sdk_version=_sdk_version(), + ) result = AgentEvalResult( run_id=run_id, tasks=task_list, trials=trial_list, scores=scores, summary=AgentEvalSummary.from_scores(scores, tasks=task_list), - benchmark=benchmark, + metadata=metadata, ) if runtime_config.output_dir is not None: @@ -686,11 +698,38 @@ def _is_completions_endpoint(url: str) -> bool: return path.endswith("/completions") and not path.endswith("/chat/completions") -def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]: - benchmarks = sorted({str(task.metadata.get("benchmark")) for task in tasks if task.metadata.get("benchmark")}) - if not benchmarks: - return {} - return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks} +def _sdk_version() -> str | None: + try: + return package_version("nemo-evaluator-sdk") + except PackageNotFoundError: # pragma: no cover - only when running from an uninstalled tree + return None + + +def _describe_target( + target: AgentEvalTarget | None, + params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, +) -> RunnerInfo: + """Identify what produced the trials, for the run's provenance. + + Runners identify themselves via the required :meth:`AgentTaskRunner.runner_info`; trials supplied + directly have no runner. + + Models and agents are described by name *and* the settings they were invoked with — the endpoint + ``url``, plus the whole ``params`` object (temperature, max_tokens, reasoning effort, system prompt, + retries, ...). A name alone is not an identity: the same model name served from two different URLs, + or at two different temperatures, would otherwise record identical provenance. ``params`` is dumped + whole rather than cherry-picked, because a filtered subset is what bites you later when the omitted + field turns out to be the one that mattered. It carries no credentials — ``Model.api_key_secret`` is + a reference on the model, and ``default_headers`` is excluded from serialization. + """ + if target is None: + return RunnerInfo(name="imported", kind="imported") + if isinstance(target, (Model, AgentBase)): + config: dict[str, Any] = {"url": getattr(target, "url", None)} + if params is not None: + config["params"] = params.model_dump(mode="json", exclude_none=True) + return RunnerInfo(name=target.name, kind="model" if isinstance(target, Model) else "agent", config=config) + return target.runner_info() def _persist_with_optional_dashboard( diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py index b2b820e0e7..82377a118d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py @@ -20,7 +20,7 @@ def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalRes path = Path(output_dir) path.mkdir(parents=True, exist_ok=True) - _write_json(path / "benchmark.json", result.benchmark) + _write_json(path / "metadata.json", result.metadata) _write_jsonl(path / "tasks.jsonl", result.tasks) _write_trials(path / "trials.jsonl", result.trials, base=path) _write_jsonl(path / "scores.jsonl", result.scores) @@ -37,7 +37,7 @@ def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: "output_dir": str(result.output_dir) if result.output_dir is not None else None, "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, "artifacts": { - "benchmark": "benchmark.json", + "metadata": "metadata.json", "tasks": "tasks.jsonl", "trials": "trials.jsonl", "scores": "scores.jsonl", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py index 7eee3f6335..46e2c07fb6 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/results.py @@ -7,12 +7,12 @@ import math from collections.abc import Sequence +from datetime import datetime from pathlib import Path -from typing import Any from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_evaluator_sdk.metrics.protocol import MetricOutput from nemo_evaluator_sdk.metrics.utils import metric_type_name from nemo_evaluator_sdk.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore @@ -68,6 +68,30 @@ def from_scores( ) +class RunMetadata(BaseModel): + """Provenance for a run: what was evaluated, by what, and when. + + Answers "what produced this result?" — previously improvised by callers inside an untyped + ``benchmark`` dict. ``labels`` remains free-form for caller-specific tags, but the fields that + every run has are typed. + """ + + model_config = ConfigDict(extra="forbid") + + labels: dict[str, str] = Field( + default_factory=dict, + description="Caller-supplied tags for this run (e.g. benchmark, mode, backend). Free-form by design.", + ) + target: RunnerInfo | None = Field( + default=None, + description="Identity of the runner/model/agent that produced the trials; None for imported trials.", + ) + started_at: datetime | None = Field(default=None, description="UTC timestamp when the run began.") + finished_at: datetime | None = Field(default=None, description="UTC timestamp when scoring completed.") + duration_sec: float | None = Field(default=None, description="Wall-clock seconds from start to finish.") + sdk_version: str | None = Field(default=None, description="nemo-evaluator-sdk version that produced the run.") + + class AgentEvalResult(BaseModel): """Root result for a completed agent evaluation: tasks, trials, scores, summary, and bundle metadata.""" @@ -78,9 +102,9 @@ class AgentEvalResult(BaseModel): trials: list[AgentEvalTrial] = Field(description="Trials produced or imported for the run.") scores: list[AgentEvalTaskScore] = Field(description="Metric scores computed for the trials.") summary: AgentEvalSummary = Field(description="Derived rollups and coverage computed for the run.") - benchmark: dict[str, Any] = Field( - default_factory=dict, - description="Benchmark metadata recorded for the run.", + metadata: RunMetadata = Field( + default_factory=RunMetadata, + description="Run provenance: labels, target identity, timings, SDK version.", ) output_dir: Path | None = Field(default=None, description="Directory the run bundle was written to, if any.") dashboard_path: Path | None = Field(default=None, description="Path to the rendered dashboard, if written.") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/callable_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/callable_runtime.py index db524b70f1..507b221783 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/callable_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/callable_runtime.py @@ -11,7 +11,13 @@ from typing import Any from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + RunnerInfo, + callable_identity, +) from nemo_evaluator_sdk.values.evidence import CandidateEvidence @@ -53,6 +59,17 @@ def __init__( self._parallelism = parallelism self._trial_id_suffix = trial_id_suffix + def runner_info(self) -> RunnerInfo: + """Identify this runner; the agent callable itself is the result-shaping detail.""" + return RunnerInfo( + name="callable", + kind="runner", + config={ + "agent_fn": callable_identity(self._agent_fn), + "parallelism": self._parallelism, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py index 717d10d653..abef83d4a4 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py @@ -23,10 +23,20 @@ from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + RunnerInfo, + callable_identity, +) from nemo_evaluator_sdk.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor +#: Wall-clock ceiling for a single task's Codex CLI invocation — one ``process.communicate()`` covering +#: the agent's whole run on that task, not a per-request or per-turn limit. Tasks run independently, so +#: this is not a budget for the evaluation as a whole. On expiry the process is terminated and the task +#: is recorded as a failed trial; it does not abort the run. DEFAULT_CODEX_TIMEOUT_S = 600 DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" @@ -76,6 +86,23 @@ def __init__( self._process_factory = process_factory or asyncio.create_subprocess_exec self._runtime_name = runtime_name + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Codex CLI settings that shape its results. + + Uses ``runtime_name``, which subclasses already set (the Docker variant reports + ``codex_docker_cli``) and which trials are stamped with, so provenance agrees with them. + """ + return RunnerInfo( + name=self._runtime_name, + kind="runner", + config={ + "model": self._model, + "timeout_s": self._timeout_s, + "codex_bin": self._codex_bin, + "prompt_builder": callable_identity(self._prompt_builder), + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 22b4252b30..7c8cdccb17 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -21,7 +21,7 @@ from uuid import uuid4 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic_core import to_jsonable_python @@ -110,6 +110,19 @@ def __init__( self._sandbox_client_factory = sandbox_client_factory self._runner = runner + def runner_info(self) -> RunnerInfo: + """Identify this runner and the sandbox settings that shape its results.""" + return RunnerInfo( + name="docker_sandbox", + kind="runner", + config={ + "model": self._model, + "image": self._image, + "timeout_s": self._timeout_s, + "instructions": self._instructions, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py index 18ba73c905..0d19c9de41 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/container_runtime.py @@ -55,7 +55,7 @@ from nemo_evaluator_sdk.agent_eval.runtimes.sandbox.api import AsyncSandbox from nemo_evaluator_sdk.agent_eval.runtimes.sandbox.base import SandboxExecResult, SandboxProvider, SandboxSpec from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_evaluator_sdk.resolver_protocols import SecretResolver from nemo_evaluator_sdk.resolvers import LocalSecretResolver @@ -177,6 +177,22 @@ async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: self._resolved_env = env self._secrets_resolved = True + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Fabric container settings that shape its results. + + Records the provider only — never ``self._secrets``, which is persisted nowhere. + """ + return RunnerInfo( + name="fabric_container", + kind="runner", + config={ + "provider": self._provider.name, + "image": self._image, + "adapter_id": self._adapter_id(), + "skills": [skill.name for skill in self._skill_set.skills], + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index 34d56b7c7a..44bc5f24e9 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -51,7 +51,7 @@ resolve_skill_mode, ) from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_evaluator_sdk.values.evidence import ( EVIDENCE_FORMAT_ATIF, @@ -160,6 +160,43 @@ def with_skill(self, skill: AgentSkill) -> FabricAgentRuntime: """ return self.with_skills([skill]) + def _adapter_id(self) -> str: + """Harness adapter selected by the Fabric config (empty when unset).""" + harness = self._config.get("harness") if isinstance(self._config, Mapping) else None + adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None + return str(adapter_id) if adapter_id is not None else "" + + def _effective_model(self) -> str | None: + """The model a run will actually use, mirroring :meth:`_compose_config`'s precedence. + + ``_compose_config`` only overwrites the config's default model when ``self._model`` is set, so + a model supplied purely through ``config`` is what runs. Reporting ``self._model`` alone would + record ``None`` for those runs, giving two runs with *different* models identical provenance — + the one thing this metadata exists to prevent. + """ + if self._model: + return self._model + models = self._config.get("models") if isinstance(self._config, Mapping) else None + default = models.get("default") if isinstance(models, Mapping) else None + model = default.get("model") if isinstance(default, Mapping) else getattr(default, "model", None) + return str(model) if model is not None else None + + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Fabric settings that shape its results.""" + return RunnerInfo( + name=self._runtime_name, + kind="runner", + config={ + "model": self._effective_model(), + "timeout_s": self._timeout_s, + "adapter_id": self._adapter_id(), + "skills": [skill.name for skill in self._skill_set.skills], + # Off means no relay/ATIF exporter, so the run captures no trajectory evidence — a + # metric that scores trajectories sees something different. + "capture_trajectory": self._capture_trajectory, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py index ab0e2a314f..d13d22086e 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/gym_runtime.py @@ -61,7 +61,7 @@ from typing import Any from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic import BaseModel, ConfigDict, Field @@ -82,6 +82,34 @@ _LOG_TAIL_LINES = 40 +#: Substrings that mark a Hydra override key as carrying a credential. Matched case-insensitively +#: against the key half of ``+key=value``. +_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential") +#: Stand-in written in place of a redacted override value. +_REDACTED = "" + + +def _redact_env_overrides(overrides: Sequence[str]) -> list[str]: + """Redact credential-looking values from Hydra overrides before they are recorded as provenance. + + ``env_overrides`` is a free-form escape hatch forwarded verbatim to ``gym env start``, so nothing + stops a caller passing ``+model.api_key=sk-...``. ``RunnerInfo.config`` is persisted into the run + bundle, so a value that looks like a credential must not be written there. + + The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance; + knowing the value is a leak. Overrides that don't parse as ``key=value`` are kept verbatim: they + carry no value to leak. + """ + redacted: list[str] = [] + for override in overrides: + key, sep, _ = override.partition("=") + if sep and any(marker in key.casefold() for marker in _SECRET_KEY_MARKERS): + redacted.append(f"{key}={_REDACTED}") + else: + redacted.append(override) + return redacted + + def _canonical_row_hash(row: Mapping[str, Any]) -> str: """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields. @@ -332,6 +360,30 @@ class GymAgentTaskRunner: def __init__(self, *, config: GymRuntimeConfig) -> None: self._config = config + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Gym settings that shape its results. + + Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this + object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key + (see :func:`_redact_env_overrides`) rather than trusted. + """ + cfg = self._config + return RunnerInfo( + name="gym", + kind="runner", + config={ + "resources_server": cfg.resources_server, + "agent": cfg.agent, + "agent_config": cfg.agent_config, + "model_type": cfg.model_type, + "num_repeats": cfg.num_repeats, + "concurrency": cfg.concurrency, + "bind_resources_server": cfg.bind_resources_server, + "env_overrides": _redact_env_overrides(cfg.env_overrides), + "reward_key": cfg.reward_key, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py index 374f2defa6..72d94941ba 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/harbor_runtime.py @@ -55,6 +55,7 @@ AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, + RunnerInfo, standard_evidence_descriptors, ) from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult @@ -192,6 +193,18 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) +def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None: + """The agent a run will actually use, mirroring ``run_job``'s resolution order. + + ``agent_import_path`` wins when set; otherwise the built-in ``agent_name``, which itself falls back + to Harbor's ``oracle`` default. Recording the resolved value keeps two runs with different custom + agents distinguishable in provenance. + """ + if config is None: + return None + return config.agent_import_path or config.agent_name or "oracle" + + class HarborAgentTaskRunner: """An :class:`AgentTaskRunner` that runs a Harbor job, then adapts its results. @@ -236,6 +249,32 @@ def __init__( self._run_job = run_job self._reward_key = config.reward_key if config is not None else reward_key + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Harbor settings that shape its results. + + Records the *effective* agent, mirroring how ``run_job`` resolves it: ``agent_import_path`` + overrides ``agent_name`` (which itself defaults to ``oracle``). Reporting the configured + ``agent_name`` alone would give two runs using different custom agents identical provenance. + """ + config = self._config + return RunnerInfo( + name="harbor", + kind="runner", + config={ + "agent_name": config.agent_name if config is not None else None, + "agent_import_path": config.agent_import_path if config is not None else None, + "agent_model_name": config.agent_model_name if config is not None else None, + "effective_agent": _effective_harbor_agent(config), + "n_attempts": config.n_attempts if config is not None else None, + # Native mode resolves the concrete job directory inside run_tasks (the name defaults + # to a timestamp), so record the configured location rather than a not-yet-known path. + "job_dir": str(self._job_dir) if self._job_dir is not None else None, + "jobs_dir": str(config.jobs_dir) if config is not None else None, + "job_name": config.job_name if config is not None else None, + "reward_key": self._reward_key, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py index 83e5d64520..163a3b8547 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py @@ -229,8 +229,10 @@ class AgentEvalRunConfig(BaseModel): ) parallelism: int = Field(default=4, ge=1, description="Maximum number of tasks scored concurrently.") write_dashboard: bool = Field(default=True, description="Whether to render an HTML dashboard for the run.") - benchmark: dict[str, Any] = Field( + labels: dict[str, str] = Field( default_factory=dict, - description="Benchmark metadata recorded alongside the run.", + description="Caller-supplied tags recorded on the run's metadata (e.g. benchmark, mode, backend, " + "scenario). Free-form by design and never derived: nothing is inferred from task metadata, so a " + "label is present only if the caller set it.", ) fail_fast: bool = Field(default=False, description="Stop the run on the first scoring failure when True.") diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py index 26918d4ed8..378373cc4a 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/trials.py @@ -101,6 +101,45 @@ async def run_tasks( config: AgentEvalRunConfig | None = None, ) -> Sequence[AgentEvalTrial]: ... + def runner_info(self) -> RunnerInfo: + """Identify this runner and the settings that shape its results, for run provenance. + + Required: every run has a producer, and the result records it on + ``AgentEvalResult.metadata.target`` so a run can be understood after the fact. Return a stable + short ``name`` (``"gym"``, ``"harbor"``) rather than a class name. ``config`` must not contain + secrets — it is persisted with the run bundle. + """ + ... + + +class RunnerInfo(BaseModel): + """Identity of whatever produced a run's trials, recorded for provenance.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Identifier of the runner/target, e.g. 'gym', 'harbor', or a model name.") + kind: str = Field( + default="runner", + description="What produced the trials: 'runner', 'model', 'agent', or 'imported' for stored trials.", + ) + version: str | None = Field(default=None, description="Version of the backing tool, when known.") + config: dict[str, Any] = Field( + default_factory=dict, + description="Runner-specific settings that affect results, recorded so a run can be understood " + "after the fact. Must not contain secrets.", + ) + + +def callable_identity(target: object) -> str: + """Module-qualified identity of a callable, for :attr:`RunnerInfo.config`. + + A bare ``__qualname__`` is ambiguous across modules — two runs using different callables that + share a name would record identical provenance — so qualify it with the defining module. + """ + module = getattr(target, "__module__", None) + name = getattr(target, "__qualname__", None) or type(target).__name__ + return f"{module}.{name}" if module else name + @runtime_checkable class AgentTrialSerde(Protocol): diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py index 26036d16e6..da9d5d0553 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_docker_example.py @@ -10,7 +10,7 @@ import pytest from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.execution.samples import build_metric_input from nemo_evaluator_sdk.values.evidence import CandidateEvidence, EvidenceDescriptor @@ -25,6 +25,9 @@ class _FakeCodexRuntime: def __init__(self, workspace: Path) -> None: self._workspace = workspace + def runner_info(self) -> RunnerInfo: + return RunnerInfo(name="fake_codex", kind="runner") + async def run_tasks( self, tasks: list[AgentEvalTask], diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py index e4caa233b0..f221924518 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_evaluator.py @@ -28,7 +28,7 @@ SemanticView, ViewSignal, ) -from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_evaluator_sdk.agent_inference import AgentInferenceContext, AgentInvocationResult, AgentInvocationStatus from nemo_evaluator_sdk.enums import AgentFormat, ModelFormat from nemo_evaluator_sdk.metrics.protocol import ( @@ -266,6 +266,9 @@ class _TaskRunner: def __init__(self) -> None: self.config: AgentEvalRunConfig | None = None + def runner_info(self) -> RunnerInfo: + return RunnerInfo(name="test_runner", kind="runner") + async def run_tasks( self, tasks: Sequence[AgentEvalTask], @@ -307,7 +310,7 @@ async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path: assert result.dashboard_path == tmp_path / "report.html" assert (tmp_path / "run.json").exists() assert (tmp_path / "scores.jsonl").exists() - assert "run_id" not in json.loads((tmp_path / "benchmark.json").read_text(encoding="utf-8")) + assert "run_id" not in json.loads((tmp_path / "metadata.json").read_text(encoding="utf-8")) score_payload = json.loads((tmp_path / "scores.jsonl").read_text(encoding="utf-8").splitlines()[0]) assert score_payload["id"] == f"{result.run_id}:task-1:trial-1:constant_metric" @@ -318,7 +321,7 @@ async def test_scores_imported_trials_with_metric_and_persists_bundle(tmp_path: run_payload = json.loads((tmp_path / "run.json").read_text(encoding="utf-8")) assert run_payload == { "artifacts": { - "benchmark": "benchmark.json", + "metadata": "metadata.json", "scores": "scores.jsonl", "summary": "summary.json", "tasks": "tasks.jsonl", diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py index b1a83d9bdd..eed52ca583 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_persistence.py @@ -70,7 +70,6 @@ def test_persist_run_writes_bundle_relative_refs_that_survive_a_move(tmp_path: P trials=[_trial_with_workspace(str(workspace))], # absolute ref under the bundle scores=[], summary=AgentEvalSummary.from_scores([], tasks=[]), - benchmark={}, ) persist_run(result, bundle) @@ -113,7 +112,6 @@ def test_persist_and_read_keep_external_evidence_refs_absolute(tmp_path: Path) - trials=[_trial_with_workspace(external_ref)], scores=[], summary=AgentEvalSummary.from_scores([], tasks=[]), - benchmark={}, ) persist_run(result, bundle) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py new file mode 100644 index 0000000000..cdea5624bc --- /dev/null +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_run_metadata.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run provenance: target identity via the runner contract, and timings.""" + +from __future__ import annotations + +from typing import cast + +from nemo_evaluator_sdk.agent_eval.evaluator import _describe_target +from nemo_evaluator_sdk.agent_eval.runtimes.sandbox.base import SandboxProvider +from nemo_evaluator_sdk.agent_eval.trials import AgentTaskRunner, RunnerInfo +from nemo_evaluator_sdk.values import Model + + +class _Runner: + """A minimal runner fulfilling the AgentTaskRunner contract.""" + + def runner_info(self) -> RunnerInfo: + return RunnerInfo(name="gym", kind="runner", version="1.2.3", config={"resources_server": "mcqa"}) + + async def run_tasks(self, tasks, config=None): # pragma: no cover - not exercised here + return [] + + +class _RunnerMissingInfo: + """A would-be runner without runner_info — no longer an AgentTaskRunner.""" + + async def run_tasks(self, tasks, config=None): # pragma: no cover - not exercised here + return [] + + +def test_runner_contract_is_satisfied_and_used() -> None: + runner = _Runner() + assert isinstance(runner, AgentTaskRunner) + + info = _describe_target(runner) + assert (info.name, info.kind, info.version) == ("gym", "runner", "1.2.3") + assert info.config == {"resources_server": "mcqa"} + + +def test_runner_info_is_required_by_the_runner_contract() -> None: + # runner_info is part of AgentTaskRunner, not an optional add-on: a class without it is not a + # runner, so the evaluator won't dispatch to it and provenance can never be missing. + assert not isinstance(_RunnerMissingInfo(), AgentTaskRunner) + + +def test_every_shipped_runner_reports_a_stable_name_and_result_shaping_config() -> None: + """All eight shipped runners: a curated name (not a class name) and the settings that change results. + + Provenance that omits a result-shaping setting is worse than none — two runs that behaved + differently would record identical metadata — so assert each runner surfaces its own knobs. + """ + from pathlib import Path + + from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner + from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexCliAgentRuntime, CodexDockerCliAgentRuntime + from nemo_evaluator_sdk.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.container_runtime import FabricContainerRuntime + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime + from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig + from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig + + async def _agent_fn(task): # pragma: no cover - never called + return None + + class _Provider: + """Identity-only stub: runner_info reads `provider.name` and nothing else, so the rest of the + SandboxProvider protocol is deliberately unimplemented and cast at the call site.""" + + name = "docker" + + harness = {"harness": {"adapter_id": "nvidia.fabric.codex"}} + runners = [ + (CallableAgentTaskRunner(_agent_fn), "callable", {"agent_fn", "parallelism"}), + (CodexCliAgentRuntime(), "codex_cli", {"model", "timeout_s", "codex_bin", "prompt_builder"}), + (CodexDockerCliAgentRuntime(), "codex_docker_cli", {"model", "timeout_s", "codex_bin", "prompt_builder"}), + (DockerSandboxAgentRuntime(), "docker_sandbox", {"model", "image", "timeout_s", "instructions"}), + ( + FabricAgentRuntime(config=harness), + "fabric", + {"model", "timeout_s", "adapter_id", "skills", "capture_trajectory"}, + ), + ( + FabricContainerRuntime(config=harness, provider=cast(SandboxProvider, _Provider())), + "fabric_container", + {"provider", "image", "adapter_id", "skills"}, + ), + ( + GymAgentTaskRunner( + config=GymRuntimeConfig(gym_root=Path("/x"), agent="a", agent_config="c", resources_server="r") + ), + "gym", + { + "resources_server", + "agent", + "model_type", + "num_repeats", + "bind_resources_server", + "env_overrides", + "reward_key", + }, + ), + ( + HarborAgentTaskRunner(config=HarborRuntimeConfig(jobs_dir=Path("/jobs"))), + "harbor", + {"agent_name", "agent_import_path", "effective_agent", "n_attempts", "jobs_dir", "reward_key"}, + ), + ] + + for runner, expected_name, expected_config_keys in runners: + info = runner.runner_info() + assert info.name == expected_name, f"{type(runner).__name__} reported {info.name!r}" + assert info.kind == "runner" + missing = expected_config_keys - set(info.config) + assert not missing, f"{expected_name} omits result-shaping config: {sorted(missing)}" + + +def test_provider_identity_is_stable_not_a_repr() -> None: + # str(provider) yields a memory address, so two identical runs would record different metadata. + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.container_runtime import FabricContainerRuntime + + class _Provider: + """Identity-only stub; see the note in the shipped-runners test above.""" + + name = "docker" + + provider = cast(SandboxProvider, _Provider()) + info = FabricContainerRuntime(config={"harness": {"adapter_id": "x"}}, provider=provider).runner_info() + assert info.config["provider"] == "docker" + assert "0x" not in info.config["provider"] + + +def test_fabric_records_a_config_supplied_model_not_just_an_explicit_one() -> None: + # _compose_config only overrides the config's default model when `model=` was passed explicitly, so + # a config-supplied model is what actually runs. Reporting the constructor arg alone would give two + # runs with different models identical provenance. + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime + + config = {"harness": {"adapter_id": "nvidia.fabric.codex"}, "models": {"default": {"model": "gpt-from-config"}}} + + assert FabricAgentRuntime(config=config).runner_info().config["model"] == "gpt-from-config" + # An explicit model still wins — that is the precedence _compose_config applies. + assert FabricAgentRuntime(config=config, model="gpt-explicit").runner_info().config["model"] == "gpt-explicit" + assert FabricAgentRuntime(config={"harness": {"adapter_id": "x"}}).runner_info().config["model"] is None + + +def test_fabric_records_whether_trajectory_evidence_was_captured() -> None: + # With capture off, no relay/ATIF exporter runs and the trial carries no trajectory evidence — so a + # trajectory-scoring metric sees something different. Both modes must be distinguishable afterwards. + from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime + + config = {"harness": {"adapter_id": "nvidia.fabric.codex"}} + assert FabricAgentRuntime(config=config, capture_trajectory=True).runner_info().config["capture_trajectory"] is True + assert ( + FabricAgentRuntime(config=config, capture_trajectory=False).runner_info().config["capture_trajectory"] is False + ) + + +def test_model_target_and_imported_trials_are_identified() -> None: + model = _describe_target(Model(name="gpt-x", url="https://example/v1/chat/completions")) + assert (model.name, model.kind) == ("gpt-x", "model") + + imported = _describe_target(None) # trials supplied directly, no target ran + assert (imported.name, imported.kind) == ("imported", "imported") + + +def test_harbor_records_the_effective_agent_when_a_custom_import_path_overrides_the_name() -> None: + # run_job uses agent_import_path when set and ignores agent_name, so recording agent_name alone + # would give two runs with different custom agents identical provenance. + from pathlib import Path + + from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborAgentTaskRunner, HarborRuntimeConfig + + def _info(**kwargs): + return HarborAgentTaskRunner(config=HarborRuntimeConfig(jobs_dir=Path("/jobs"), **kwargs)).runner_info().config + + custom = _info(agent_import_path="pkg_a:Agent", agent_model_name="model-a") + assert custom["effective_agent"] == "pkg_a:Agent" + assert custom["agent_model_name"] == "model-a" + + other = _info(agent_import_path="pkg_b:Agent", agent_model_name="model-b") + assert other["effective_agent"] != custom["effective_agent"] + + # Built-in agents still resolve through agent_name, defaulting to Harbor's oracle. + assert _info(agent_name="oracle")["effective_agent"] == "oracle" + assert _info()["effective_agent"] == "oracle" + + +def test_gym_redacts_credential_looking_env_overrides() -> None: + # env_overrides is a free-form Hydra escape hatch forwarded to `gym env start`, and RunnerInfo.config + # is persisted into the run bundle — so a value that looks like a credential must not be written there. + from pathlib import Path + + from nemo_evaluator_sdk.agent_eval.runtimes.gym_runtime import GymAgentTaskRunner, GymRuntimeConfig + + runner = GymAgentTaskRunner( + config=GymRuntimeConfig( + gym_root=Path("/x"), + agent="a", + agent_config="c", + resources_server="r", + env_overrides=[ + "+model.api_key=sk-should-not-be-recorded", + "+env.HF_TOKEN=hf_should-not-be-recorded", + "+agent.temperature=0.7", + "--flagged-without-a-value", + ], + ) + ) + + recorded = runner.runner_info().config["env_overrides"] + + assert recorded == [ + "+model.api_key=", + "+env.HF_TOKEN=", + "+agent.temperature=0.7", # not credential-shaped, kept verbatim for reproducibility + "--flagged-without-a-value", # no value to leak + ] + assert not any("sk-" in entry or "hf_" in entry for entry in recorded) + + +def test_model_provenance_records_the_endpoint_and_invocation_params() -> None: + # A name alone is not an identity: the same model served from two URLs, or run at two different + # temperatures, would otherwise record identical provenance. + from nemo_evaluator_sdk.values import RunConfigOnlineModel + from nemo_evaluator_sdk.values.params import InferenceParams + + params = RunConfigOnlineModel(inference=InferenceParams(temperature=0.0, max_tokens=256), max_retries=5) + info = _describe_target(Model(name="gpt-x", url="https://a.example/v1/chat/completions"), params) + + assert (info.name, info.kind) == ("gpt-x", "model") + assert info.config["url"] == "https://a.example/v1/chat/completions" + assert info.config["params"]["inference"] == {"temperature": 0.0, "max_tokens": 256} + assert info.config["params"]["max_retries"] == 5 + + # Same name, different endpoint -> distinguishable. + other = _describe_target(Model(name="gpt-x", url="https://b.example/v1/chat/completions"), params) + assert other.config["url"] != info.config["url"] + + +def _load_example(name: str): + """Import an example runtime module. + + The example runtimes use relative imports, so they are imported as a package (rather than loaded + by path like the standalone codex_docker example) with the SDK package root on sys.path. + """ + import importlib + import sys + from pathlib import Path + + root = str(Path(__file__).resolve().parents[2]) + if root not in sys.path: + sys.path.insert(0, root) + return importlib.import_module(f"examples.run_agent_eval.{name}") + + +def _example_runtimes() -> list[object]: + aut = _load_example("aut_runtime") + platform = _load_example("platform_runtime") + workflow = _load_example("workflow_runtime") + return [ + workflow.WorkflowAgentRuntime(), + aut.NatAutRuntime(aut.AutConfig(aut_agent_name="an-agent")), + platform.NatWorkflowRuntime(), + ] + + +def test_example_runtimes_still_satisfy_the_runner_contract() -> None: + """Making runner_info required silently breaks any runner that lacks it. + + `AgentTaskRunner` is runtime_checkable and isinstance-dispatched, so a runner without + `runner_info` stops matching and the evaluator raises "unsupported agent-eval target type" — at + run time, with nothing at import time to warn you. The shipped runtimes were updated; the example + runtimes were missed once already, so assert on those too. + """ + for runtime in _example_runtimes(): + assert isinstance(runtime, AgentTaskRunner), f"{type(runtime).__name__} is no longer an AgentTaskRunner" + assert runtime.runner_info().name, f"{type(runtime).__name__} reported an empty name" + + +def test_example_runner_provenance_excludes_the_api_keys_its_config_carries() -> None: + # AutConfig and NatWorkflowConfig hold nvidia/anthropic API keys alongside their result-shaping + # settings, and RunnerInfo.config is persisted with the run — so these must enumerate fields + # rather than dump the config. + aut_module = _load_example("aut_runtime") + platform_module = _load_example("platform_runtime") + + aut = aut_module.NatAutRuntime( + aut_module.AutConfig( + aut_agent_name="an-agent", + nvidia_api_key="nvapi-secret", + inference_nvidia_api_key="nvapi-secret-2", + anthropic_api_key="sk-ant-secret", + ) + ).runner_info() + workflow = platform_module.NatWorkflowRuntime( + platform_module.NatWorkflowConfig(nvidia_api_key="nvapi-secret") + ).runner_info() + + for info in (aut, workflow): + assert "secret" not in str(info.config), f"{info.name} leaked a credential into provenance" + assert not any("api_key" in key for key in info.config) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 008e1338a7..e463b3f190 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -171,7 +171,7 @@ async def to_spec( trials=submit_spec.trials, max_concurrent_tasks=submit_spec.max_concurrent_tasks, fail_fast=submit_spec.fail_fast, - benchmark=submit_spec.benchmark, + labels=submit_spec.labels, ) @classmethod @@ -314,7 +314,7 @@ def run( params=params, prompt_template=prompt_template, parallelism=spec.max_concurrent_tasks, - benchmark=spec.benchmark, + labels=spec.labels, fail_fast=spec.fail_fast, write_dashboard=False, ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py index afd688cf08..a073bbbc96 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_spec.py @@ -232,7 +232,10 @@ class _AgentEvalSpecCommon(BaseModel): "`params.parallelism`, which bounds concurrent inference requests *within* trial generation.", ) fail_fast: bool = Field(default=False, description="Stop the run on the first scoring failure when True.") - benchmark: dict[str, Any] = Field(default_factory=dict, description="Benchmark metadata recorded with the run.") + labels: dict[str, str] = Field( + default_factory=dict, + description="Caller-supplied tags recorded on the run's metadata (e.g. benchmark, mode, backend).", + ) @model_validator(mode="after") def _require_exactly_one_trial_source(self) -> Self: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index 453ac48ab0..8bb80c814f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -12,6 +12,8 @@ from collections import defaultdict from collections.abc import Awaitable, Callable, Sequence from datetime import UTC, datetime +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as package_version from logging import getLogger from pathlib import Path from typing import Any, cast, overload @@ -21,7 +23,7 @@ import nemo_platform.beta.evaluator.inference as inference from nemo_platform.beta.evaluator.agent_eval.dashboard import write_dashboard from nemo_platform.beta.evaluator.agent_eval.persistence import persist_run -from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary +from nemo_platform.beta.evaluator.agent_eval.results import AgentEvalResult, AgentEvalSummary, RunMetadata from nemo_platform.beta.evaluator.agent_eval.scores import ( AgentEvalDiagnostic, AgentEvalDiagnosticSeverity, @@ -35,6 +37,7 @@ AgentEvalTrialStatus, AgentOutput, AgentTaskRunner, + RunnerInfo, ) from nemo_platform.beta.evaluator.agent_inference import ( AgentInferenceContext, @@ -155,6 +158,7 @@ async def run( run_id = resolved_config.run_id or _new_run_id() runtime_config = resolved_config.model_copy(update={"run_id": run_id}) + started_at = datetime.now(UTC) # Branch on which seam was supplied so the type checker can narrow ``target`` to a # concrete ``AgentEvalTarget`` without a cast. @@ -172,14 +176,22 @@ async def run( config=runtime_config, run_id=run_id, ) - benchmark = {**_benchmark_metadata(task_list), **runtime_config.benchmark} + finished_at = datetime.now(UTC) + metadata = RunMetadata( + labels=dict(runtime_config.labels), + target=_describe_target(target, runtime_config.params), + started_at=started_at, + finished_at=finished_at, + duration_sec=(finished_at - started_at).total_seconds(), + sdk_version=_sdk_version(), + ) result = AgentEvalResult( run_id=run_id, tasks=task_list, trials=trial_list, scores=scores, summary=AgentEvalSummary.from_scores(scores, tasks=task_list), - benchmark=benchmark, + metadata=metadata, ) if runtime_config.output_dir is not None: @@ -686,11 +698,38 @@ def _is_completions_endpoint(url: str) -> bool: return path.endswith("/completions") and not path.endswith("/chat/completions") -def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]: - benchmarks = sorted({str(task.metadata.get("benchmark")) for task in tasks if task.metadata.get("benchmark")}) - if not benchmarks: - return {} - return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks} +def _sdk_version() -> str | None: + try: + return package_version("nemo-evaluator-sdk") + except PackageNotFoundError: # pragma: no cover - only when running from an uninstalled tree + return None + + +def _describe_target( + target: AgentEvalTarget | None, + params: RunConfig | RunConfigOnline | RunConfigOnlineModel | None = None, +) -> RunnerInfo: + """Identify what produced the trials, for the run's provenance. + + Runners identify themselves via the required :meth:`AgentTaskRunner.runner_info`; trials supplied + directly have no runner. + + Models and agents are described by name *and* the settings they were invoked with — the endpoint + ``url``, plus the whole ``params`` object (temperature, max_tokens, reasoning effort, system prompt, + retries, ...). A name alone is not an identity: the same model name served from two different URLs, + or at two different temperatures, would otherwise record identical provenance. ``params`` is dumped + whole rather than cherry-picked, because a filtered subset is what bites you later when the omitted + field turns out to be the one that mattered. It carries no credentials — ``Model.api_key_secret`` is + a reference on the model, and ``default_headers`` is excluded from serialization. + """ + if target is None: + return RunnerInfo(name="imported", kind="imported") + if isinstance(target, (Model, AgentBase)): + config: dict[str, Any] = {"url": getattr(target, "url", None)} + if params is not None: + config["params"] = params.model_dump(mode="json", exclude_none=True) + return RunnerInfo(name=target.name, kind="model" if isinstance(target, Model) else "agent", config=config) + return target.runner_info() def _persist_with_optional_dashboard( diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py index b9c26a04bf..d5eb90c9fa 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/persistence.py @@ -20,7 +20,7 @@ def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalRes path = Path(output_dir) path.mkdir(parents=True, exist_ok=True) - _write_json(path / "benchmark.json", result.benchmark) + _write_json(path / "metadata.json", result.metadata) _write_jsonl(path / "tasks.jsonl", result.tasks) _write_trials(path / "trials.jsonl", result.trials, base=path) _write_jsonl(path / "scores.jsonl", result.scores) @@ -37,7 +37,7 @@ def _run_manifest(result: AgentEvalResult) -> dict[str, Any]: "output_dir": str(result.output_dir) if result.output_dir is not None else None, "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None, "artifacts": { - "benchmark": "benchmark.json", + "metadata": "metadata.json", "tasks": "tasks.jsonl", "trials": "trials.jsonl", "scores": "scores.jsonl", diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py index 897e30aeea..ca142bc795 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/results.py @@ -7,12 +7,12 @@ import math from collections.abc import Sequence +from datetime import datetime from pathlib import Path -from typing import Any from nemo_platform.beta.evaluator.agent_eval.scores import AgentEvalScoreStatus, AgentEvalTaskScore from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalTask, SemanticReducer, ViewSignal -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, RunnerInfo from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput from nemo_platform.beta.evaluator.metrics.utils import metric_type_name from nemo_platform.beta.evaluator.values.results import AggregatedMetricResult, AggregateRangeScore, AggregateScore @@ -68,6 +68,30 @@ def from_scores( ) +class RunMetadata(BaseModel): + """Provenance for a run: what was evaluated, by what, and when. + + Answers "what produced this result?" — previously improvised by callers inside an untyped + ``benchmark`` dict. ``labels`` remains free-form for caller-specific tags, but the fields that + every run has are typed. + """ + + model_config = ConfigDict(extra="forbid") + + labels: dict[str, str] = Field( + default_factory=dict, + description="Caller-supplied tags for this run (e.g. benchmark, mode, backend). Free-form by design.", + ) + target: RunnerInfo | None = Field( + default=None, + description="Identity of the runner/model/agent that produced the trials; None for imported trials.", + ) + started_at: datetime | None = Field(default=None, description="UTC timestamp when the run began.") + finished_at: datetime | None = Field(default=None, description="UTC timestamp when scoring completed.") + duration_sec: float | None = Field(default=None, description="Wall-clock seconds from start to finish.") + sdk_version: str | None = Field(default=None, description="nemo-evaluator-sdk version that produced the run.") + + class AgentEvalResult(BaseModel): """Root result for a completed agent evaluation: tasks, trials, scores, summary, and bundle metadata.""" @@ -78,9 +102,9 @@ class AgentEvalResult(BaseModel): trials: list[AgentEvalTrial] = Field(description="Trials produced or imported for the run.") scores: list[AgentEvalTaskScore] = Field(description="Metric scores computed for the trials.") summary: AgentEvalSummary = Field(description="Derived rollups and coverage computed for the run.") - benchmark: dict[str, Any] = Field( - default_factory=dict, - description="Benchmark metadata recorded for the run.", + metadata: RunMetadata = Field( + default_factory=RunMetadata, + description="Run provenance: labels, target identity, timings, SDK version.", ) output_dir: Path | None = Field(default=None, description="Directory the run bundle was written to, if any.") dashboard_path: Path | None = Field(default=None, description="Path to the rendered dashboard, if written.") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py index 1a36ac4868..d6af7e43be 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/callable_runtime.py @@ -11,7 +11,13 @@ from typing import Any from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + RunnerInfo, + callable_identity, +) from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence @@ -53,6 +59,17 @@ def __init__( self._parallelism = parallelism self._trial_id_suffix = trial_id_suffix + def runner_info(self) -> RunnerInfo: + """Identify this runner; the agent callable itself is the result-shaping detail.""" + return RunnerInfo( + name="callable", + kind="runner", + config={ + "agent_fn": callable_identity(self._agent_fn), + "parallelism": self._parallelism, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py index 32bacefc36..ec8cb76601 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py @@ -23,10 +23,20 @@ from nemo_platform.beta.evaluator.agent_eval.runtimes.docker_sandbox import DockerSandboxAgentRuntime from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import ( + AgentEvalTrial, + AgentEvalTrialStatus, + AgentOutput, + RunnerInfo, + callable_identity, +) from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor +#: Wall-clock ceiling for a single task's Codex CLI invocation — one ``process.communicate()`` covering +#: the agent's whole run on that task, not a per-request or per-turn limit. Tasks run independently, so +#: this is not a budget for the evaluation as a whole. On expiry the process is terminated and the task +#: is recorded as a failed trial; it does not abort the run. DEFAULT_CODEX_TIMEOUT_S = 600 DEFAULT_CODEX_DOCKER_MODEL = "gpt-5.4" DEFAULT_CODEX_DOCKER_CLI_IMAGE = "node:22-alpine" @@ -76,6 +86,23 @@ def __init__( self._process_factory = process_factory or asyncio.create_subprocess_exec self._runtime_name = runtime_name + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Codex CLI settings that shape its results. + + Uses ``runtime_name``, which subclasses already set (the Docker variant reports + ``codex_docker_cli``) and which trials are stamped with, so provenance agrees with them. + """ + return RunnerInfo( + name=self._runtime_name, + kind="runner", + config={ + "model": self._model, + "timeout_s": self._timeout_s, + "codex_bin": self._codex_bin, + "prompt_builder": callable_identity(self._prompt_builder), + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index 9113164c00..6ac195d5a2 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -21,7 +21,7 @@ from uuid import uuid4 from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic_core import to_jsonable_python @@ -110,6 +110,19 @@ def __init__( self._sandbox_client_factory = sandbox_client_factory self._runner = runner + def runner_info(self) -> RunnerInfo: + """Identify this runner and the sandbox settings that shape its results.""" + return RunnerInfo( + name="docker_sandbox", + kind="runner", + config={ + "model": self._model, + "image": self._image, + "timeout_s": self._timeout_s, + "instructions": self._instructions, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py index 59fda190e6..b145ae6c5c 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/container_runtime.py @@ -55,7 +55,7 @@ from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.api import AsyncSandbox from nemo_platform.beta.evaluator.agent_eval.runtimes.sandbox.base import SandboxExecResult, SandboxProvider, SandboxSpec from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_platform.beta.evaluator.resolver_protocols import SecretResolver from nemo_platform.beta.evaluator.resolvers import LocalSecretResolver @@ -177,6 +177,22 @@ async def resolve_secrets(self, secret_resolver: SecretResolver) -> None: self._resolved_env = env self._secrets_resolved = True + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Fabric container settings that shape its results. + + Records the provider only — never ``self._secrets``, which is persisted nowhere. + """ + return RunnerInfo( + name="fabric_container", + kind="runner", + config={ + "provider": self._provider.name, + "image": self._image, + "adapter_id": self._adapter_id(), + "skills": [skill.name for skill in self._skill_set.skills], + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index 77423a4547..b132d53014 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -51,7 +51,7 @@ resolve_skill_mode, ) from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_platform.beta.evaluator.agent_eval.workspace_seeds import SEED_FILES_INPUT_KEY, seed_workspace from nemo_platform.beta.evaluator.values.evidence import ( EVIDENCE_FORMAT_ATIF, @@ -160,6 +160,43 @@ def with_skill(self, skill: AgentSkill) -> FabricAgentRuntime: """ return self.with_skills([skill]) + def _adapter_id(self) -> str: + """Harness adapter selected by the Fabric config (empty when unset).""" + harness = self._config.get("harness") if isinstance(self._config, Mapping) else None + adapter_id = harness.get("adapter_id") if isinstance(harness, Mapping) else None + return str(adapter_id) if adapter_id is not None else "" + + def _effective_model(self) -> str | None: + """The model a run will actually use, mirroring :meth:`_compose_config`'s precedence. + + ``_compose_config`` only overwrites the config's default model when ``self._model`` is set, so + a model supplied purely through ``config`` is what runs. Reporting ``self._model`` alone would + record ``None`` for those runs, giving two runs with *different* models identical provenance — + the one thing this metadata exists to prevent. + """ + if self._model: + return self._model + models = self._config.get("models") if isinstance(self._config, Mapping) else None + default = models.get("default") if isinstance(models, Mapping) else None + model = default.get("model") if isinstance(default, Mapping) else getattr(default, "model", None) + return str(model) if model is not None else None + + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Fabric settings that shape its results.""" + return RunnerInfo( + name=self._runtime_name, + kind="runner", + config={ + "model": self._effective_model(), + "timeout_s": self._timeout_s, + "adapter_id": self._adapter_id(), + "skills": [skill.name for skill in self._skill_set.skills], + # Off means no relay/ATIF exporter, so the run captures no trajectory evidence — a + # metric that scores trajectories sees something different. + "capture_trajectory": self._capture_trajectory, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py index 94526bae68..ef7f75ccde 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/gym_runtime.py @@ -61,7 +61,7 @@ from typing import Any from nemo_platform.beta.evaluator.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask -from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput +from nemo_platform.beta.evaluator.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, RunnerInfo from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_platform.beta.evaluator.values.evidence import CandidateEvidence, EvidenceDescriptor from pydantic import BaseModel, ConfigDict, Field @@ -82,6 +82,34 @@ _LOG_TAIL_LINES = 40 +#: Substrings that mark a Hydra override key as carrying a credential. Matched case-insensitively +#: against the key half of ``+key=value``. +_SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password", "passwd", "credential") +#: Stand-in written in place of a redacted override value. +_REDACTED = "" + + +def _redact_env_overrides(overrides: Sequence[str]) -> list[str]: + """Redact credential-looking values from Hydra overrides before they are recorded as provenance. + + ``env_overrides`` is a free-form escape hatch forwarded verbatim to ``gym env start``, so nothing + stops a caller passing ``+model.api_key=sk-...``. ``RunnerInfo.config`` is persisted into the run + bundle, so a value that looks like a credential must not be written there. + + The *key* is always kept — knowing that a run overrode ``model.api_key`` is useful provenance; + knowing the value is a leak. Overrides that don't parse as ``key=value`` are kept verbatim: they + carry no value to leak. + """ + redacted: list[str] = [] + for override in overrides: + key, sep, _ = override.partition("=") + if sep and any(marker in key.casefold() for marker in _SECRET_KEY_MARKERS): + redacted.append(f"{key}={_REDACTED}") + else: + redacted.append(override) + return redacted + + def _canonical_row_hash(row: Mapping[str, Any]) -> str: """Stable ``sha256`` of a Gym dataset row, excluding runtime-injected fields. @@ -332,6 +360,30 @@ class GymAgentTaskRunner: def __init__(self, *, config: GymRuntimeConfig) -> None: self._config = config + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Gym settings that shape its results. + + Credentials normally live in the Gym checkout's gitignored ``env.yaml`` and never reach this + object — but ``env_overrides`` is a free-form escape hatch, so its values are redacted by key + (see :func:`_redact_env_overrides`) rather than trusted. + """ + cfg = self._config + return RunnerInfo( + name="gym", + kind="runner", + config={ + "resources_server": cfg.resources_server, + "agent": cfg.agent, + "agent_config": cfg.agent_config, + "model_type": cfg.model_type, + "num_repeats": cfg.num_repeats, + "concurrency": cfg.concurrency, + "bind_resources_server": cfg.bind_resources_server, + "env_overrides": _redact_env_overrides(cfg.env_overrides), + "reward_key": cfg.reward_key, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py index 20d56349d3..5683ac0719 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/harbor_runtime.py @@ -55,6 +55,7 @@ AgentEvalTrial, AgentEvalTrialStatus, AgentOutput, + RunnerInfo, standard_evidence_descriptors, ) from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult @@ -192,6 +193,18 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: return MetricResult(outputs=[MetricOutput(name=self._output_name, value=value)]) +def _effective_harbor_agent(config: HarborRuntimeConfig | None) -> str | None: + """The agent a run will actually use, mirroring ``run_job``'s resolution order. + + ``agent_import_path`` wins when set; otherwise the built-in ``agent_name``, which itself falls back + to Harbor's ``oracle`` default. Recording the resolved value keeps two runs with different custom + agents distinguishable in provenance. + """ + if config is None: + return None + return config.agent_import_path or config.agent_name or "oracle" + + class HarborAgentTaskRunner: """An :class:`AgentTaskRunner` that runs a Harbor job, then adapts its results. @@ -236,6 +249,32 @@ def __init__( self._run_job = run_job self._reward_key = config.reward_key if config is not None else reward_key + def runner_info(self) -> RunnerInfo: + """Identify this runner and the Harbor settings that shape its results. + + Records the *effective* agent, mirroring how ``run_job`` resolves it: ``agent_import_path`` + overrides ``agent_name`` (which itself defaults to ``oracle``). Reporting the configured + ``agent_name`` alone would give two runs using different custom agents identical provenance. + """ + config = self._config + return RunnerInfo( + name="harbor", + kind="runner", + config={ + "agent_name": config.agent_name if config is not None else None, + "agent_import_path": config.agent_import_path if config is not None else None, + "agent_model_name": config.agent_model_name if config is not None else None, + "effective_agent": _effective_harbor_agent(config), + "n_attempts": config.n_attempts if config is not None else None, + # Native mode resolves the concrete job directory inside run_tasks (the name defaults + # to a timestamp), so record the configured location rather than a not-yet-known path. + "job_dir": str(self._job_dir) if self._job_dir is not None else None, + "jobs_dir": str(config.jobs_dir) if config is not None else None, + "job_name": config.job_name if config is not None else None, + "reward_key": self._reward_key, + }, + ) + async def run_tasks( self, tasks: Sequence[AgentEvalTask], diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index 6c0da4ac21..e2b377c886 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -229,8 +229,10 @@ class AgentEvalRunConfig(BaseModel): ) parallelism: int = Field(default=4, ge=1, description="Maximum number of tasks scored concurrently.") write_dashboard: bool = Field(default=True, description="Whether to render an HTML dashboard for the run.") - benchmark: dict[str, Any] = Field( + labels: dict[str, str] = Field( default_factory=dict, - description="Benchmark metadata recorded alongside the run.", + description="Caller-supplied tags recorded on the run's metadata (e.g. benchmark, mode, backend, " + "scenario). Free-form by design and never derived: nothing is inferred from task metadata, so a " + "label is present only if the caller set it.", ) fail_fast: bool = Field(default=False, description="Stop the run on the first scoring failure when True.") diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py index bd63644032..17faef3584 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/trials.py @@ -101,6 +101,45 @@ async def run_tasks( config: AgentEvalRunConfig | None = None, ) -> Sequence[AgentEvalTrial]: ... + def runner_info(self) -> RunnerInfo: + """Identify this runner and the settings that shape its results, for run provenance. + + Required: every run has a producer, and the result records it on + ``AgentEvalResult.metadata.target`` so a run can be understood after the fact. Return a stable + short ``name`` (``"gym"``, ``"harbor"``) rather than a class name. ``config`` must not contain + secrets — it is persisted with the run bundle. + """ + ... + + +class RunnerInfo(BaseModel): + """Identity of whatever produced a run's trials, recorded for provenance.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(description="Identifier of the runner/target, e.g. 'gym', 'harbor', or a model name.") + kind: str = Field( + default="runner", + description="What produced the trials: 'runner', 'model', 'agent', or 'imported' for stored trials.", + ) + version: str | None = Field(default=None, description="Version of the backing tool, when known.") + config: dict[str, Any] = Field( + default_factory=dict, + description="Runner-specific settings that affect results, recorded so a run can be understood " + "after the fact. Must not contain secrets.", + ) + + +def callable_identity(target: object) -> str: + """Module-qualified identity of a callable, for :attr:`RunnerInfo.config`. + + A bare ``__qualname__`` is ambiguous across modules — two runs using different callables that + share a name would record identical provenance — so qualify it with the defining module. + """ + module = getattr(target, "__module__", None) + name = getattr(target, "__qualname__", None) or type(target).__name__ + return f"{module}.{name}" if module else name + @runtime_checkable class AgentTrialSerde(Protocol):