feat(evaluator): refresh evaluator plugin skills

docs/evaluator/agent-eval/index.mdx

@@ -13,10 +13,12 @@ got there*. Each task carries its own metrics, so a single suite can grade heter
 
 <Note>
 
-**Platform-plugin support is in progress.** The pages in this section run agent evaluations through
-the **local SDK** (`AgentEvaluator().run()`). Running them through the **NeMo Platform plugin** as
-durable platform jobs — the way [dataset-driven metrics](/documentation/evaluate-models/metrics)
-already can — is under active development; for now, use the local SDK path shown here.
+**Local interfaces and durable platform interfaces are available.**
+- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs
+that do not require running nemo-platform.
+- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline
+tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces
+above remain dataset-driven only.
 
 </Note>
 
@@ -84,9 +86,10 @@ print(result.summary)
   - the **trajectory** — how the agent worked (its tool use and steps);
   - **views** — named roll-ups you define on a task that combine two or more of its metric outputs into one reported score (for example, averaging an accuracy metric and a tool-use metric into a single `quality` score);
   - the **run-level aggregate** — results also roll up across the whole run.
-- **Runs locally.** A full run — including the `report.html` dashboard — is produced on your machine
-  with no platform services required. (Running the same suite as a durable platform job through the
-  evaluator plugin is in progress — see the note above.)
+- **Runs locally or durably.** A local run can produce the full bundle,
+  including `report.html`, without platform services. Use the plugin's
+  `agent-evaluate` job when the task suite needs durable platform execution and
+  persisted result metadata.
 - **Measurement, not decisions.** The evaluator produces scores, aggregates, and provenance — it
   doesn't decide pass/fail, gate a release, or compare runs. Those decisions belong to whatever
   consumes the results.

docs/evaluator/index.mdx

@@ -120,10 +120,12 @@ result = job.get_result()
 
 <Note>
 
-**Agent evaluation runs from the SDK today.** Task-driven runs use the local SDK
-(`AgentEvaluator().run()`); running them as durable platform jobs — the way dataset-driven metrics
-already can — is [in progress](/documentation/evaluate-models/agent-eval). Use the local SDK path for
-now.
+**Agent evaluation has local interfaces and durable platform interfaces.**
+- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs
+that do not require running nemo-platform.
+- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline
+tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces
+above remain dataset-driven only.
 
 </Note>
 

packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py

@@ -9,7 +9,7 @@
 ``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``:
 
 * **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` —
-  and takes codex-specific ``harness.settings`` (sandbox mode, git-repo check, ...).
+  and takes codex-specific ``harness.settings`` such as sandbox and approval modes.
 * **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and
   declares its ``input``/``output`` schemas instead.
 
@@ -28,10 +28,11 @@
 import json
 
 from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
-from nemo_fabric import (  # ty: ignore[unresolved-import]
+from nemo_fabric import (
     FabricConfig,
     HarnessConfig,
     MetadataConfig,
+    ModelConfig,
     RuntimeConfig,
 )
 
@@ -40,17 +41,17 @@
     metadata=MetadataConfig(name="codex-eval"),
     harness=HarnessConfig(
         adapter_id="nvidia.fabric.codex",
-        settings={"sandbox": "read-only", "skip_git_repo_check": True},
+        settings={"sandbox": "read-only"},
     ),
-    models={"default": {"provider": "openai", "model": "gpt-5.4"}},
+    models={"default": ModelConfig(provider="openai", model="gpt-5.4")},
     runtime=RuntimeConfig.from_mapping({"mode": "oneshot", "transport": "cli"}),
 )
 
 #: Hermes SDK harness — in-library transport, explicit chat/message schemas.
 HERMES_SDK_CONFIG = FabricConfig(
     metadata=MetadataConfig(name="hermes-eval"),
     harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
-    models={"default": {"provider": "nvidia", "model": "qwen2.5-coder-32b"}},
+    models={"default": ModelConfig(provider="nvidia", model="qwen2.5-coder-32b")},
     runtime=RuntimeConfig.from_mapping(
         {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"}
     ),
@@ -65,7 +66,7 @@
 
 def build_runtime(harness: str, *, model: str | None = None, work_root: str | None = None) -> FabricAgentRuntime:
     """Build a :class:`FabricAgentRuntime` for a named harness (see :data:`HARNESS_CONFIGS`)."""
-    return FabricAgentRuntime(config=HARNESS_CONFIGS[harness], model=model, work_root=work_root)
+    return FabricAgentRuntime(config=HARNESS_CONFIGS[harness].to_mapping(), model=model, work_root=work_root)
 
 
 def main() -> None:

packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py

@@ -8,8 +8,9 @@
   in CI: it proves the runner -> evaluator -> metric -> evidence chain, i.e. the metric receives and
   reads the trajectory (ATIF) evidence for the task.
 - ``test_fabric_codex_live_eval_captures_atif_trajectory`` is the real fabric->codex->Relay run, gated
-  behind the required binaries so CI skips it; run it locally after ``uv sync --extra fabric``
-  plus ``script/dev-install-fabric.sh`` for the relay gateway.
+  behind the required binaries so CI skips it; run it locally after
+  ``uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact`` plus
+  ``script/dev-install-fabric.sh`` for the relay gateway.
 """
 
 from __future__ import annotations
@@ -187,16 +188,16 @@ def __init__(self, **kwargs: Any) -> None:
             self.__dict__.update(kwargs)
 
     module = types.ModuleType("nemo_fabric")
-    module.Fabric = _FakeClient  # type: ignore[attr-defined]
-    module.FabricConfig = _FakeConfig  # type: ignore[attr-defined]
-    module.EnvironmentConfig = _FakeEnvironment  # type: ignore[attr-defined]
-    module.ModelConfig = _FakeModelConfig  # type: ignore[attr-defined]
-    module.RunRequest = _FakeRunRequest  # type: ignore[attr-defined]
+    setattr(module, "Fabric", _FakeClient)
+    setattr(module, "FabricConfig", _FakeConfig)
+    setattr(module, "EnvironmentConfig", _FakeEnvironment)
+    setattr(module, "ModelConfig", _FakeModelConfig)
+    setattr(module, "RunRequest", _FakeRunRequest)
     # The runtime builds the relay observability config from Fabric's own typed models (lazy import).
-    module.RelayObservabilityConfig = _FakeRelayModel  # type: ignore[attr-defined]
-    module.RelayAtifConfig = _FakeRelayModel  # type: ignore[attr-defined]
-    module.RelayAtofConfig = _FakeRelayModel  # type: ignore[attr-defined]
-    module.RelayAtofFileSinkConfig = _FakeRelayModel  # type: ignore[attr-defined]
+    setattr(module, "RelayObservabilityConfig", _FakeRelayModel)
+    setattr(module, "RelayAtifConfig", _FakeRelayModel)
+    setattr(module, "RelayAtofConfig", _FakeRelayModel)
+    setattr(module, "RelayAtofFileSinkConfig", _FakeRelayModel)
     monkeypatch.setitem(sys.modules, "nemo_fabric", module)
     # nemo_relay stays a hard (installed) dependency here so ``run_tasks``'s capture-trajectory fail-fast
     # (``import nemo_relay.observability``) resolves; only the optional native nemo_fabric SDK is faked.
@@ -215,8 +216,10 @@ def __init__(self, **kwargs: Any) -> None:
     trial = result.trials[0]
     assert trial.status == "completed"
     # The trajectory is exposed under the standard trace key, as an existing ATIF file.
+    assert trial.evidence is not None
     trace = trial.evidence.descriptors[EVIDENCE_TRACE]
     assert trace.format == EVIDENCE_FORMAT_ATIF
+    assert trace.ref is not None
     assert Path(trace.ref).exists()
     # The metric received the evidence and scored from the trajectory content.
     scores = [s for s in result.scores if s.metric_type == "has-trajectory"]
@@ -242,13 +245,14 @@ def _codex_adapter_installed() -> bool:
 
 
 # No NeMo-Fabric checkout in the gate: the adapter registry resolves from the installed wheels
-# (<sys.prefix>/share/nemo-fabric/adapters), so `uv sync --extra fabric` is enough.
+# (<sys.prefix>/share/nemo-fabric/adapters), so the package-scoped `fabric` extra is enough.
 _LIVE_READY = bool(shutil.which("codex") and shutil.which("nemo-relay") and _codex_adapter_installed())
 _LIVE_MODEL = os.environ.get("NEMO_FABRIC_LIVE_MODEL", "gpt-5.6-terra")
 requires_live_fabric = pytest.mark.skipif(
     not _LIVE_READY,
     reason=(
-        "needs the harness adapters (uv sync --extra fabric) + the nemo-relay gateway "
+        "needs the harness adapters "
+        "(uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact) + the nemo-relay gateway "
         "(script/dev-install-fabric.sh) + codex on PATH"
     ),
 )
@@ -263,9 +267,15 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None
         "harness": {
             "adapter_id": "nvidia.fabric.codex",
             "resolution": "preinstalled",
-            "settings": {"sandbox": "workspace-write", "skip_git_repo_check": True, "timeout_seconds": 180},
+            "settings": {"sandbox": "workspace-write"},
+        },
+        "runtime": {
+            "mode": "oneshot",
+            "transport": "cli",
+            "input_schema": "text",
+            "output_schema": "message",
+            "timeout_seconds": 180,
         },
-        "runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"},
         "environment": {"provider": "local", "workspace": str(tmp_path / "ws")},
         # Fabric's codex adapter requires an explicit model provider — it does not fall back to the
         # Codex CLI's own configured default, and starting without one fails the adapter lifecycle
@@ -288,8 +298,10 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None
 
     trial = result.trials[0]
     assert trial.status == "completed", trial.metadata
+    assert trial.evidence is not None
     trace = trial.evidence.descriptors[EVIDENCE_TRACE]
     assert trace.format == EVIDENCE_FORMAT_ATIF
+    assert trace.ref is not None
     atif = Path(trace.ref)
     assert atif.exists() and atif.stat().st_size > 0
     assert "steps" in json.loads(atif.read_text(encoding="utf-8"))