refactor(evaluator)!: make persisting an agent-eval run an explicit call

packages/nemo_evaluator_sdk/examples/agentic_eval_with_fabric.ipynb

@@ -599,13 +599,16 @@
     "result = evaluator.run_sync(\n",
     "    tasks=suite.tasks,\n",
     "    target=target,\n",
-    "    config=AgentEvalRunConfig(output_dir=OUTPUT_DIR, write_dashboard=True, parallelism=3),\n",
+    "    config=AgentEvalRunConfig(work_dir=OUTPUT_DIR, parallelism=3),\n",
     ")\n",
     "\n",
+    "# Storing the run is its own step; it defaults to the work_dir the config named.\n",
+    "location = result.persist()\n",
+    "\n",
     "print(\"run_id    :\", result.run_id)\n",
     "print(\"tasks     :\", result.summary.task_count)\n",
     "print(\"trials    :\", result.summary.trial_count)\n",
-    "print(\"dashboard :\", result.dashboard_path)"
+    "print(\"dashboard :\", location.dashboard_path)"
    ]
   },
   {
@@ -685,7 +688,7 @@
    "cell_type": "markdown",
    "id": "cell-34",
    "metadata": {},
-   "source": "To see **what the agent actually did** — its output, the files it changed, its step-by-step trajectory\n— open the HTML dashboard at `result.dashboard_path`, or read the persisted bundle under\n`result.output_dir` (`trials.jsonl`, `scores.jsonl`, `summary.json`). The trial evidence (the final\nworkspace and the ATIF trace) is what the metrics above opened to score each run."
+   "source": "To see **what the agent actually did** — its output, the files it changed, its step-by-step trajectory\n— open the HTML dashboard at `location.dashboard_path`, or read the persisted bundle under\n`location.output_dir` (`trials.jsonl`, `scores.jsonl`, `summary.json`). The trial evidence (the final\nworkspace and the ATIF trace) is what the metrics above opened to score each run."
   },
   {
    "cell_type": "markdown",

packages/nemo_evaluator_sdk/examples/codex_docker/example.py

@@ -32,7 +32,7 @@
 
 from nemo_evaluator_sdk import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
 from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
-from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, BundleLocation
 from nemo_evaluator_sdk.agent_eval.runtimes.codex.runtime import CodexDockerCliAgentRuntime
 from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalRunConfig, AgentEvalTask
 from nemo_evaluator_sdk.agent_eval.trials import AgentTaskRunner
@@ -108,8 +108,12 @@ async def evaluate(
     output_dir: str | Path | None = None,
     runtime: AgentTaskRunner | None = None,
     write_dashboard: bool = True,
-) -> AgentEvalResult:
-    """Run one Docker Codex task and score its host-readable workspace evidence."""
+) -> tuple[AgentEvalResult, BundleLocation]:
+    """Run one Docker Codex task, score its workspace evidence, and store the run.
+
+    Returns the result and where it was written: ``run`` itself no longer persists, so storing is an
+    explicit step here.
+    """
     resolved_output_dir = Path(output_dir).expanduser() if output_dir is not None else _new_output_dir()
     target = runtime or _docker_runtime(resolved_output_dir)
 
@@ -127,21 +131,21 @@ async def evaluate(
         metrics=[WorkspaceArtifactMetric()],
     )
 
-    return await AgentEvaluator().run(
+    result = await AgentEvaluator().run(
         tasks=[task],
         target=target,
         config=AgentEvalRunConfig(
-            output_dir=resolved_output_dir,
+            work_dir=resolved_output_dir,
             parallelism=1,
-            write_dashboard=write_dashboard,
             benchmark={"name": "codex-docker-evidence-sanity"},
         ),
     )
+    return result, result.persist(write_dashboard=write_dashboard)
 
 
 async def main() -> None:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s")
-    result = await evaluate()
+    result, location = await evaluate()
 
     trial = result.trials[0]
     if trial.output is None or trial.evidence is None:
@@ -156,7 +160,7 @@ async def main() -> None:
     print(f"artifact contents: {artifact.read_text(encoding='utf-8').strip()}")
     print(f"workspace_artifact.output_matches: {scores['workspace_artifact.output_matches']}")
     print(f"workspace_artifact.artifact_matches: {scores['workspace_artifact.artifact_matches']}")
-    print(f"run bundle: {result.output_dir}")
+    print(f"run bundle: {location.output_dir}")
 
 
 if __name__ == "__main__":

packages/nemo_evaluator_sdk/examples/fabric_container/run_e2e.py

@@ -63,7 +63,7 @@ async def main() -> int:
     )
 
     output_dir = Path(os.environ.get("FABRIC_OUTPUT_DIR", "/tmp/fabric-container-e2e"))
-    (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(output_dir=output_dir))
+    (trial,) = await runtime.run_tasks([task], AgentEvalRunConfig(work_dir=output_dir))
 
     print("=== TRIAL ===")
     print("status:", trial.status)

packages/nemo_evaluator_sdk/examples/gym/run_gym_eval.py

@@ -97,15 +97,18 @@ async def _main(args: argparse.Namespace) -> int:
     result = await AgentEvaluator().run(
         tasks=tasks,
         target=runner,
-        config=AgentEvalRunConfig(output_dir=output_dir, parallelism=1),
+        config=AgentEvalRunConfig(work_dir=output_dir, parallelism=1),
     )
+    # Storing the run is its own step. Defaults to the run's work_dir, so the bundle contains the
+    # evidence the trials point at.
+    location = result.persist()
 
     print("=== RESULT ===")
     print(f"tasks: {result.summary.task_count}  trials: {result.summary.trial_count}")
     print("aggregate scores:")
     for aggregate in result.summary.scores.scores:
         print(f"  {aggregate.name}: mean={aggregate.mean}")
-    print(f"\nRun bundle (run.json, trials.jsonl, scores.jsonl, report.html): {output_dir}")
+    print(f"\nRun bundle (run.json, trials.jsonl, scores.jsonl, report.html): {location.output_dir}")
     return 0
 
 

packages/nemo_evaluator_sdk/examples/profbench/runner.py

@@ -121,13 +121,14 @@ async def run_profbench_mode(
         trials=trials,
         target=target,
         config=AgentEvalRunConfig(
-            output_dir=output_dir,
+            work_dir=output_dir,
             run_id=f"{run_instance_id}-{mode.value}",
             params=params,
             benchmark=benchmark_meta,
-            write_dashboard=False,
         ),
     )
+    # This example renders its own dashboards below, so persistence skips the built-in one.
+    result.persist(write_dashboard=False)
     sdk_dashboard_path, dashboard_path = write_example_dashboards(result, output_dir)
 
     overall = _profbench_overall(result)

packages/nemo_evaluator_sdk/examples/run_agent_eval/pipeline.py

@@ -74,7 +74,7 @@ async def run_tasks(
             target=target,
             config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
         )
-        self._maybe_write_gate(result)
+        self._persist_and_gate(result, output_dir)
         return result
 
     async def score_trials(
@@ -93,7 +93,7 @@ async def score_trials(
             trials=list(trials),
             config=self._run_config(output_dir=output_dir, run_id=run_id, benchmark=benchmark),
         )
-        self._maybe_write_gate(result)
+        self._persist_and_gate(result, output_dir)
         return result
 
     def _run_config(
@@ -104,10 +104,9 @@ def _run_config(
         benchmark: dict[str, object] | None,
     ) -> AgentEvalRunConfig:
         return AgentEvalRunConfig(
-            output_dir=output_dir,
+            work_dir=output_dir,
             run_id=run_id,
             parallelism=self.config.parallelism,
-            write_dashboard=self.config.write_dashboard,
             benchmark=dict(benchmark or {}),
         )
 
@@ -122,16 +121,20 @@ def _with_extra_metrics(self, task: AgentEvalTask) -> AgentEvalTask:
             return task
         return task.model_copy(update={"metrics": metrics + appended})
 
-    def _maybe_write_gate(self, result: AgentEvalResult) -> None:
-        if not (self.config.write_gate and result.output_dir is not None):
+    def _persist_and_gate(self, result: AgentEvalResult, output_dir: Path | None) -> None:
+        """Store the run (when a directory was given) and write the gate report beside it."""
+        if output_dir is None:
+            return
+        location = result.persist(write_dashboard=self.config.write_dashboard)
+        if not self.config.write_gate:
             return
         baseline = (
             load_baseline_summary(self.config.baseline_summary_path)
             if self.config.baseline_summary_path is not None
             else None
         )
         report = evaluate_gate(result, thresholds=self.config.gate_thresholds, baseline_summary=baseline)
-        write_gate_report(report, result.output_dir)
+        write_gate_report(report, location.output_dir)
 
 
 __all__ = [

packages/nemo_evaluator_sdk/examples/run_agent_eval/platform_runtime.py

@@ -105,7 +105,7 @@ def task_image_tag(task_id: str) -> str:
 
 def resolve_run_layout(task: AgentEvalTask, config: AgentEvalRunConfig | None) -> AgenticRunLayout:
     """Resolve/create the on-disk layout for one task run."""
-    output_dir = config.output_dir if config is not None else None
+    output_dir = config.work_dir if config is not None else None
     run_dir = resolve_run_dir(output_dir, lambda: Path.cwd() / "nat-jobs" / task.id) / task.id
     base = prepare_run_layout(run_dir, str(task.inputs.get("instruction") or task.intent))
     state_dir = base.run_dir / "state"

packages/nemo_evaluator_sdk/examples/run_agent_eval/run_agent_eval.py

@@ -159,9 +159,9 @@ def _print_result(result: AgentEvalResult) -> None:
         if score.mean is not None:
             print(f"  {score.name}: mean={score.mean:.3f}")
     _print_measurements(result)
-    if result.output_dir is not None:
-        print(f"output_dir: {result.output_dir}")
-        print(f"gate: {result.output_dir / 'gate.json'}")
+    if result.work_dir is not None:
+        print(f"work_dir: {result.work_dir}")
+        print(f"gate: {result.work_dir / 'gate.json'}")
 
 
 def _print_measurements(result: AgentEvalResult) -> None:

packages/nemo_evaluator_sdk/examples/run_agent_eval/workflow_runtime.py

@@ -179,7 +179,7 @@ def _format_command(self, instruction_path: Path, workspace_dir: Path, input_jso
         return [substitutions.get(token, token) for token in self.config.command]
 
     def _run_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path:
-        root = (config.output_dir or Path.cwd()) / "evidence" / RUNTIME_NAME
+        root = (config.work_dir or Path.cwd()) / "evidence" / RUNTIME_NAME
         return root / (_safe_name(task.id) or f"task-{index}")
 
 

packages/nemo_evaluator_sdk/examples/skill_eval/run_skill_eval.py

@@ -258,15 +258,17 @@ async def _main() -> int:
         baseline = await AgentEvaluator().run(
             tasks=tasks,
             target=baseline_runtime,
-            config=AgentEvalRunConfig(run_id="baseline", output_dir=output_dir / "baseline", write_dashboard=False),
+            config=AgentEvalRunConfig(run_id="baseline", work_dir=output_dir / "baseline"),
         )
+        baseline.persist(write_dashboard=False)
         treated = await AgentEvaluator().run(
             tasks=tasks,
             target=baseline_runtime.with_skill(
                 skill
             ),  # We include the skill in the treated arm, so the two runs differ in *exactly* the skill.
-            config=AgentEvalRunConfig(run_id="treated", output_dir=output_dir / "treated", write_dashboard=False),
+            config=AgentEvalRunConfig(run_id="treated", work_dir=output_dir / "treated"),
         )
+        treated.persist(write_dashboard=False)
     except SkillInjectionError as exc:
         print(f"skill eval failed to load the bundled skill: {exc}")
         return 1

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py

@@ -19,8 +19,6 @@
 
 import httpx
 import nemo_evaluator_sdk.inference as inference
-from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard
-from nemo_evaluator_sdk.agent_eval.persistence import persist_run
 from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, AgentEvalSummary
 from nemo_evaluator_sdk.agent_eval.scores import (
     AgentEvalDiagnostic,
@@ -180,10 +178,9 @@ async def run(
             scores=scores,
             summary=AgentEvalSummary.from_scores(scores, tasks=task_list),
             benchmark=benchmark,
+            work_dir=runtime_config.work_dir,
         )
 
-        if runtime_config.output_dir is not None:
-            result = _persist_with_optional_dashboard(result, runtime_config.output_dir, runtime_config.write_dashboard)
         return result
 
     def run_sync(
@@ -324,8 +321,8 @@ async def generate_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
                         "invocation_id": f"{config.run_id}:{task.id}:{target.name}",
                     }
                     evidence_dir = (
-                        _task_evidence_dir(Path(config.output_dir), index=index, task_id=task.id)
-                        if config.output_dir is not None and isinstance(target, AgentBase)
+                        _task_evidence_dir(Path(config.work_dir), index=index, task_id=task.id)
+                        if config.work_dir is not None and isinstance(target, AgentBase)
                         else None
                     )
                     resolved_inference_fn = self.inference_fn
@@ -693,18 +690,6 @@ def _benchmark_metadata(tasks: list[AgentEvalTask]) -> dict[str, Any]:
     return {"benchmark": benchmarks[0] if len(benchmarks) == 1 else benchmarks}
 
 
-def _persist_with_optional_dashboard(
-    result: AgentEvalResult,
-    output_dir: Path,
-    write_html: bool,
-) -> AgentEvalResult:
-    path = Path(output_dir)
-    dashboard_path = None
-    if write_html:
-        dashboard_path = write_dashboard(result.model_copy(update={"output_dir": path}), path / "report.html")
-    return persist_run(result.model_copy(update={"output_dir": path, "dashboard_path": dashboard_path}), path)
-
-
 def _new_run_id() -> str:
     timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M%S")
     return f"agent-eval-{timestamp}-{uuid.uuid4().hex[:8]}"

packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/persistence.py

@@ -10,32 +10,52 @@
 from pathlib import Path
 from typing import Any
 
-from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult
+from nemo_evaluator_sdk.agent_eval.dashboard import write_dashboard
+from nemo_evaluator_sdk.agent_eval.results import AgentEvalResult, BundleLocation
 from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial
 from pydantic import BaseModel
 
+#: Filename of the rendered HTML dashboard inside a bundle.
+DASHBOARD_FILENAME = "report.html"
 
-def persist_run(result: AgentEvalResult, output_dir: str | Path) -> AgentEvalResult:
-    """Persist a completed run bundle to ``output_dir``."""
+
+def persist_run(
+    result: AgentEvalResult,
+    output_dir: str | Path,
+    *,
+    write_html_dashboard: bool = True,
+) -> BundleLocation:
+    """Write a completed run to a bundle at ``output_dir`` and report where it landed.
+
+    Explicit rather than a side effect of :meth:`AgentEvaluator.run`: computing an evaluation and
+    storing one are different decisions, and folding them together is what forced the result object to
+    carry paths it could not know at construction time. (Same reasoning as ``publish_to_intake``.)
+
+    Set ``write_html_dashboard=False`` to skip rendering ``report.html`` — the dashboard is written
+    here so the manifest can record it in a single pass.
+    """
     path = Path(output_dir)
     path.mkdir(parents=True, exist_ok=True)
 
+    # Render first so the manifest below can name it; the dashboard reads only the run's own contents.
+    dashboard_path = write_dashboard(result, path / DASHBOARD_FILENAME) if write_html_dashboard else None
+
     _write_json(path / "benchmark.json", result.benchmark)
     _write_jsonl(path / "tasks.jsonl", result.tasks)
     _write_trials(path / "trials.jsonl", result.trials, base=path)
     _write_jsonl(path / "scores.jsonl", result.scores)
     _write_json(path / "summary.json", result.summary)
 
-    updated = result.model_copy(update={"output_dir": path})
-    _write_json(path / "run.json", _run_manifest(updated))
-    return updated
+    location = BundleLocation(output_dir=path, dashboard_path=dashboard_path)
+    _write_json(path / "run.json", _run_manifest(result, location))
+    return location
 
 
-def _run_manifest(result: AgentEvalResult) -> dict[str, Any]:
+def _run_manifest(result: AgentEvalResult, location: BundleLocation) -> dict[str, Any]:
     return {
         "run_id": result.run_id,
-        "output_dir": str(result.output_dir) if result.output_dir is not None else None,
-        "dashboard_path": str(result.dashboard_path) if result.dashboard_path is not None else None,
+        "output_dir": str(location.output_dir),
+        "dashboard_path": str(location.dashboard_path) if location.dashboard_path is not None else None,
         "artifacts": {
             "benchmark": "benchmark.json",
             "tasks": "tasks.jsonl",