diff --git a/docs/evaluator/agent-eval/index.mdx b/docs/evaluator/agent-eval/index.mdx
index 377f7ceb9d..b095bdcf01 100644
--- a/docs/evaluator/agent-eval/index.mdx
+++ b/docs/evaluator/agent-eval/index.mdx
@@ -13,10 +13,12 @@ got there*. Each task carries its own metrics, so a single suite can grade heter
-**Platform-plugin support is in progress.** The pages in this section run agent evaluations through
-the **local SDK** (`AgentEvaluator().run()`). Running them through the **NeMo Platform plugin** as
-durable platform jobs — the way [dataset-driven metrics](/documentation/evaluate-models/metrics)
-already can — is under active development; for now, use the local SDK path shown here.
+**Local interfaces and durable platform interfaces are available.**
+- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs
+that do not require running nemo-platform.
+- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline
+tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces
+above remain dataset-driven only.
@@ -84,9 +86,10 @@ print(result.summary)
- the **trajectory** — how the agent worked (its tool use and steps);
- **views** — named roll-ups you define on a task that combine two or more of its metric outputs into one reported score (for example, averaging an accuracy metric and a tool-use metric into a single `quality` score);
- the **run-level aggregate** — results also roll up across the whole run.
-- **Runs locally.** A full run — including the `report.html` dashboard — is produced on your machine
- with no platform services required. (Running the same suite as a durable platform job through the
- evaluator plugin is in progress — see the note above.)
+- **Runs locally or durably.** A local run can produce the full bundle,
+ including `report.html`, without platform services. Use the plugin's
+ `agent-evaluate` job when the task suite needs durable platform execution and
+ persisted result metadata.
- **Measurement, not decisions.** The evaluator produces scores, aggregates, and provenance — it
doesn't decide pass/fail, gate a release, or compare runs. Those decisions belong to whatever
consumes the results.
diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx
index ab124840ad..c21494fb3f 100644
--- a/docs/evaluator/index.mdx
+++ b/docs/evaluator/index.mdx
@@ -120,10 +120,12 @@ result = job.get_result()
-**Agent evaluation runs from the SDK today.** Task-driven runs use the local SDK
-(`AgentEvaluator().run()`); running them as durable platform jobs — the way dataset-driven metrics
-already can — is [in progress](/documentation/evaluate-models/agent-eval). Use the local SDK path for
-now.
+**Agent evaluation has local interfaces and durable platform interfaces.**
+- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs
+that do not require running nemo-platform.
+- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline
+tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces
+above remain dataset-driven only.
diff --git a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py
index e2e47385d9..a499f4fe61 100644
--- a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py
+++ b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py
@@ -9,7 +9,7 @@
``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``:
* **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` —
- and takes codex-specific ``harness.settings`` (sandbox mode, git-repo check, ...).
+ and takes codex-specific ``harness.settings`` such as sandbox and approval modes.
* **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and
declares its ``input``/``output`` schemas instead.
@@ -28,10 +28,11 @@
import json
from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime
-from nemo_fabric import ( # ty: ignore[unresolved-import]
+from nemo_fabric import (
FabricConfig,
HarnessConfig,
MetadataConfig,
+ ModelConfig,
RuntimeConfig,
)
@@ -40,9 +41,9 @@
metadata=MetadataConfig(name="codex-eval"),
harness=HarnessConfig(
adapter_id="nvidia.fabric.codex",
- settings={"sandbox": "read-only", "skip_git_repo_check": True},
+ settings={"sandbox": "read-only"},
),
- models={"default": {"provider": "openai", "model": "gpt-5.4"}},
+ models={"default": ModelConfig(provider="openai", model="gpt-5.4")},
runtime=RuntimeConfig.from_mapping({"mode": "oneshot", "transport": "cli"}),
)
@@ -50,7 +51,7 @@
HERMES_SDK_CONFIG = FabricConfig(
metadata=MetadataConfig(name="hermes-eval"),
harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"),
- models={"default": {"provider": "nvidia", "model": "qwen2.5-coder-32b"}},
+ models={"default": ModelConfig(provider="nvidia", model="qwen2.5-coder-32b")},
runtime=RuntimeConfig.from_mapping(
{"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"}
),
@@ -65,7 +66,7 @@
def build_runtime(harness: str, *, model: str | None = None, work_root: str | None = None) -> FabricAgentRuntime:
"""Build a :class:`FabricAgentRuntime` for a named harness (see :data:`HARNESS_CONFIGS`)."""
- return FabricAgentRuntime(config=HARNESS_CONFIGS[harness], model=model, work_root=work_root)
+ return FabricAgentRuntime(config=HARNESS_CONFIGS[harness].to_mapping(), model=model, work_root=work_root)
def main() -> None:
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
index 9c12c223af..fd75eca7f8 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
@@ -8,8 +8,9 @@
in CI: it proves the runner -> evaluator -> metric -> evidence chain, i.e. the metric receives and
reads the trajectory (ATIF) evidence for the task.
- ``test_fabric_codex_live_eval_captures_atif_trajectory`` is the real fabric->codex->Relay run, gated
- behind the required binaries so CI skips it; run it locally after ``uv sync --extra fabric``
- plus ``script/dev-install-fabric.sh`` for the relay gateway.
+ behind the required binaries so CI skips it; run it locally after
+ ``uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact`` plus
+ ``script/dev-install-fabric.sh`` for the relay gateway.
"""
from __future__ import annotations
@@ -187,16 +188,16 @@ def __init__(self, **kwargs: Any) -> None:
self.__dict__.update(kwargs)
module = types.ModuleType("nemo_fabric")
- module.Fabric = _FakeClient # type: ignore[attr-defined]
- module.FabricConfig = _FakeConfig # type: ignore[attr-defined]
- module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined]
- module.ModelConfig = _FakeModelConfig # type: ignore[attr-defined]
- module.RunRequest = _FakeRunRequest # type: ignore[attr-defined]
+ setattr(module, "Fabric", _FakeClient)
+ setattr(module, "FabricConfig", _FakeConfig)
+ setattr(module, "EnvironmentConfig", _FakeEnvironment)
+ setattr(module, "ModelConfig", _FakeModelConfig)
+ setattr(module, "RunRequest", _FakeRunRequest)
# The runtime builds the relay observability config from Fabric's own typed models (lazy import).
- module.RelayObservabilityConfig = _FakeRelayModel # type: ignore[attr-defined]
- module.RelayAtifConfig = _FakeRelayModel # type: ignore[attr-defined]
- module.RelayAtofConfig = _FakeRelayModel # type: ignore[attr-defined]
- module.RelayAtofFileSinkConfig = _FakeRelayModel # type: ignore[attr-defined]
+ setattr(module, "RelayObservabilityConfig", _FakeRelayModel)
+ setattr(module, "RelayAtifConfig", _FakeRelayModel)
+ setattr(module, "RelayAtofConfig", _FakeRelayModel)
+ setattr(module, "RelayAtofFileSinkConfig", _FakeRelayModel)
monkeypatch.setitem(sys.modules, "nemo_fabric", module)
# nemo_relay stays a hard (installed) dependency here so ``run_tasks``'s capture-trajectory fail-fast
# (``import nemo_relay.observability``) resolves; only the optional native nemo_fabric SDK is faked.
@@ -215,8 +216,10 @@ def __init__(self, **kwargs: Any) -> None:
trial = result.trials[0]
assert trial.status == "completed"
# The trajectory is exposed under the standard trace key, as an existing ATIF file.
+ assert trial.evidence is not None
trace = trial.evidence.descriptors[EVIDENCE_TRACE]
assert trace.format == EVIDENCE_FORMAT_ATIF
+ assert trace.ref is not None
assert Path(trace.ref).exists()
# The metric received the evidence and scored from the trajectory content.
scores = [s for s in result.scores if s.metric_type == "has-trajectory"]
@@ -242,13 +245,14 @@ def _codex_adapter_installed() -> bool:
# No NeMo-Fabric checkout in the gate: the adapter registry resolves from the installed wheels
-# (/share/nemo-fabric/adapters), so `uv sync --extra fabric` is enough.
+# (/share/nemo-fabric/adapters), so the package-scoped `fabric` extra is enough.
_LIVE_READY = bool(shutil.which("codex") and shutil.which("nemo-relay") and _codex_adapter_installed())
_LIVE_MODEL = os.environ.get("NEMO_FABRIC_LIVE_MODEL", "gpt-5.6-terra")
requires_live_fabric = pytest.mark.skipif(
not _LIVE_READY,
reason=(
- "needs the harness adapters (uv sync --extra fabric) + the nemo-relay gateway "
+ "needs the harness adapters "
+ "(uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact) + the nemo-relay gateway "
"(script/dev-install-fabric.sh) + codex on PATH"
),
)
@@ -263,9 +267,15 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None
"harness": {
"adapter_id": "nvidia.fabric.codex",
"resolution": "preinstalled",
- "settings": {"sandbox": "workspace-write", "skip_git_repo_check": True, "timeout_seconds": 180},
+ "settings": {"sandbox": "workspace-write"},
+ },
+ "runtime": {
+ "mode": "oneshot",
+ "transport": "cli",
+ "input_schema": "text",
+ "output_schema": "message",
+ "timeout_seconds": 180,
},
- "runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"},
"environment": {"provider": "local", "workspace": str(tmp_path / "ws")},
# Fabric's codex adapter requires an explicit model provider — it does not fall back to the
# Codex CLI's own configured default, and starting without one fails the adapter lifecycle
@@ -288,8 +298,10 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None
trial = result.trials[0]
assert trial.status == "completed", trial.metadata
+ assert trial.evidence is not None
trace = trial.evidence.descriptors[EVIDENCE_TRACE]
assert trace.format == EVIDENCE_FORMAT_ATIF
+ assert trace.ref is not None
atif = Path(trace.ref)
assert atif.exists() and atif.stat().st_size > 0
assert "steps" in json.loads(atif.read_text(encoding="utf-8"))
diff --git a/plugins/nemo-evaluator/README.md b/plugins/nemo-evaluator/README.md
index bc79b6da31..0191df4808 100644
--- a/plugins/nemo-evaluator/README.md
+++ b/plugins/nemo-evaluator/README.md
@@ -1,73 +1,63 @@
# NeMo Evaluator Plugin
-A NeMo Platform plugin that brings Evaluator SDK metric execution into the
-platform.
+The Evaluator plugin connects the NeMo Evaluator SDK to NeMo Platform. It
+provides:
-The plugin exposes an `evaluator` service, CLI commands under `nemo evaluator`,
-an SDK accessor on `NeMoPlatform.evaluator`, and an `evaluator.run/evaluator.submit` for
-local plugin runs and durable platform submissions.
-
-## What it provides
-
-- **CLI** commands for plugin status, job schema inspection, local runs, and
+- **CLI** `nemo evaluator` commands for plugin status, job schema inspection, local runs, and
job submissions.
-- **Service** routes for evaluator job management.
+- **Service** routes for evaluator job management: `plugins/nemo-evaluator/src/nemo_evaluator/service.py`.
- **SDK accessor** at `client.evaluator` for status checks, local runs, job
submission, status polling, result retrieval, and artifact download.
- **Evaluator job** support for inline SDK metric specs, inline rows, and
Fileset-backed datasets.
+ - Dataset-driven `evaluator.evaluate` jobs.
+ - Task-driven `evaluator.agent-evaluate` jobs.
- **Docs and skills** that are published through the plugin entry points for
evaluator-specific reference and troubleshooting.
-## Installation (developer)
-
-Prerequisites:
+## Developer setup
-- Python and `uv` are available.
-- Commands run from the repo root.
-- `NVIDIA_API_KEY` is exported when running online or model-backed metrics.
-
-This plugin is a `uv` workspace member. From the repo root:
+This plugin is a `uv` workspace member. From the repository root:
```bash
-uv sync
+# The `make bootstrap` target creates the Python environment, syncs Python dependencies, builds Studio assets, and installs local plugins.
+make bootstrap
+source .venv/bin/activate
```
-For local platform testing, start the platform after syncing:
+Verify the installation:
```bash
-nemo services run
+nemo --help
```
-The root workspace also includes this plugin in the enabled plugin set, so the
-`nemo evaluator` CLI group should be available in the synced environment.
-
-## CLI quickstart
-
-Check that the plugin is installed:
+Check the plugin status:
```bash
-nemo evaluator info
+uv run nemo evaluator info
```
-Inspect the registered job contract:
+Follow the repository `SETUP.md` for detailed setup instructions and starting local NeMo Platform services.
-```bash
-nemo evaluator evaluate explain
-```
+## Dataset-Driven vs. Task-Driven evaluation
+Review the [Evaluator documentation](https://docs.nvidia.com/nemo-platform/documentation/evaluate-models#two-shapes-of-evaluation) for a detailed explanation of the difference between dataset-driven and task-driven evaluation.
+
+## Dataset-Driven evaluation
-Run a minimal exact-match metric from the bundled example spec:
+### CLI Commands
+Inspect the current schema and run the checked offline example:
```bash
-nemo evaluator evaluate run \
- --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json
+uv run nemo evaluator evaluate explain
+uv run nemo evaluator evaluate run \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
```
-Submit the same spec as a platform durable job:
+Submit the same spec as a durable job:
```bash
-nemo evaluator evaluate submit \
- --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json
+uv run nemo evaluator evaluate submit \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
```
The submit response includes a generated job name, for example `nemo-evaluator-zlhn1ecd`. Wait for the job to complete, then list and download its results:
@@ -79,64 +69,132 @@ nemo jobs results download aggregate-scores --job --output-file aggre
nemo jobs results download row-scores --job --output-file row-scores.jsonl
```
-## Python SDK quickstart
+See also the checked LLM-judge spec example in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json`.
-Use the mounted platform SDK accessor, `client.evaluator`:
+### Platform SDK Execution
+
+Use the mounted SDK resource to submit durable evaluation jobs:
```python
from nemo_evaluator_sdk import ExactMatchMetric, RunConfig
from nemo_platform import NeMoPlatform
-
client = NeMoPlatform(base_url="http://localhost:8080", workspace="default")
-status = client.evaluator.plugin_status()
-
metric = ExactMatchMetric(
reference="{{item.expected}}",
- candidate="{{item.model_output}}",
+ candidate="{{item.output}}",
)
dataset = [
- {"expected": "blue", "model_output": "Blue"},
- {"expected": "Jupiter", "model_output": "Saturn"},
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
]
-local_result = client.evaluator.run(
- metric=metric,
- dataset=dataset,
- config=RunConfig(parallelism=2),
-)
-
job = client.evaluator.submit(
metric=metric,
dataset=dataset,
config=RunConfig(parallelism=2),
)
+
job.wait_until_done()
-submitted_result = job.get_result()
-artifact_dir = job.download_artifacts(path="evaluation-artifacts")
+remote_result = job.get_result()
+artifact_dir = job.download_artifacts("evaluation-artifacts")
```
-## Local and remote inputs
+`submit` returns an `EvaluatorJobResource`. Always call
+`wait_until_done()` before retrieving result artifacts.
+
+## Task-Driven Agent evaluation
+
+### CLI Commands
+
+#### Durable job
+
+Inspect the task-driven job schema:
+
+```bash
+uv run nemo evaluator agent-evaluate explain
+```
+
+The checked spec gives Fabric one task and scores the runner's final response
+with exact match. Submit it as a durable platform job:
+
+```bash
+uv run nemo evaluator agent-evaluate submit \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json
+```
+
+Replace `/` and ensure the job environment includes the Fabric
+Codex adapter, Codex CLI, and its provider credentials. Set
+`capture_trajectory` to `true` only when NeMo Relay is also available.
+For repository setup, follow
+[Prepare Fabric in a repository checkout](../../skills/nemo-evaluator-plugin/SKILL.md#prepare-fabric-in-a-repository-checkout).
+
+### SDK Execution
+Plugin SDK execution is not supported for task-driven evaluation. Use the standalone Python SDK instead, which is available for local execution.
+
+#### Standalone SDK
+
+For an in-process agent callable, pass a direct `AgentTaskRunner` to the
+standalone SDK:
+
+```python
+from nemo_evaluator_sdk import ExactMatchMetric
+from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
+from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
+
+
+async def answer(task: AgentEvalTask) -> str:
+ return "Paris"
+
+
+task = AgentEvalTask(
+ id="capital-france",
+ intent="Name the capital of France.",
+ inputs={"instruction": "What is the capital of France?"},
+ reference={"expected": "Paris"},
+ metrics=[
+ ExactMatchMetric(
+ reference="{{reference.expected}}",
+ candidate="{{sample.output_text}}",
+ )
+ ],
+)
+result = AgentEvaluator().run_sync(
+ tasks=[task],
+ target=CallableAgentTaskRunner(answer),
+)
+print(result.summary)
+```
+
+See the [agent-evaluation reference](../../skills/nemo-evaluator-plugin/references/agent-evaluation.md)
+for tasksets, other durable targets, and precomputed trials.
+
+## Stored resources
+
+The SDK namespace includes:
-### Dataset support
+- `client.evaluator.metrics`
+- `client.evaluator.tasks`
+- `client.evaluator.tasksets`
+- `client.evaluator.eval_results`
+- `client.evaluator.agent_eval_results`
-- Local runs support local dataset paths, inline rows, and Fileset references.
-- Jobs support inline rows and Fileset references.
+Metrics, tasks, and tasksets support create, retrieve, list, and delete. Result
+resources support retrieve, list, and delete.
-### Model/Agent Auth
+## Authentication
-For online evaluation or LLM-as-judge evaluations, authentication depends on the
-execution mode:
+- Local model-backed evaluation resolves `api_key_secret` as a local
+ environment-variable name, such as `NVIDIA_API_KEY`..
+- Durable platformjobs resolve it as a NeMo Platform secret in the target workspace.
-- Local `nemo evaluator evaluate run` resolves `api_key_secret` as a local
- environment variable name, such as `NVIDIA_API_KEY`.
-- Remote `nemo evaluator evaluate submit` resolves `api_key_secret` as a NeMo
- Platform secret in the target workspace.
+Never place a credential value in a spec or log.
-## Next steps
+## References
-- [Evaluator plugin reference](src/nemo_evaluator/docs/index.md)
-- [Evaluator platform docs](../../docs/evaluator/index.md)
-- [Evaluator plugin skill](src/nemo_evaluator/skills/evaluator-plugin/SKILL.md)
+- [Plugin reference](src/nemo_evaluator/docs/index.md)
+- [Evaluator documentation](https://docs.nvidia.com/nemo-platform/documentation/evaluate-models)
+- [Canonical evaluator skill](../../skills/nemo-evaluator-plugin/SKILL.md)
- [Evaluator API auth](../../skills/nemo-evaluator-plugin/references/api-auth.md)
-- [Evaluation troubleshooting](../../skills/nemo-evaluator-plugin/references/troubleshooting.md)
+- [Troubleshooting](../../skills/nemo-evaluator-plugin/references/troubleshooting.md)
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py
index d74638d0e9..5ff1e28b43 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py
@@ -102,7 +102,7 @@ def info() -> None:
"plugin": self.name,
"status": "ready",
"service": "/apis/evaluator/v1/healthz",
- "jobs": ["evaluator.evaluate"],
+ "jobs": ["evaluator.evaluate", "evaluator.agent-evaluate"],
"sdk": "nemo_evaluator_sdk.Evaluator",
}
)
diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md b/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md
index c9de7e9808..1dba88f560 100644
--- a/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md
+++ b/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md
@@ -1,93 +1,156 @@
# Evaluator Plugin Reference
-The evaluator plugin is a first-party for evaluator functionality. It keeps the plugin identity separate from the legacy `/apis/evaluation` service while proving the basic surfaces needed for SDK-backed jobs.
+The Evaluator plugin connects the NeMo Evaluator SDK to NeMo Platform. Its
+command group is `nemo evaluator`; the legacy generated `nemo evaluation`
+surface is not used for new workflows.
-## Registered Surfaces
+## Registered interfaces
-| Surface | Entry point | Current behavior |
-|---|---|---|
-| CLI | `nemo.cli:evaluator` | Adds `nemo evaluator info` and hosts evaluator job commands. |
-| Service | `nemo.services:evaluator` | `jobs`, `healthz` paths. |
-| SDK | `nemo.sdk:evaluator` | Adds `client.evaluator.plugin_status() and run(), submit() interfaces`. |
-| Job | `nemo.jobs:evaluator.evaluate` | Backs local `run` through in-process execution and `submit` through durable platform job submission. |
-| Docs | `nemo.docs:evaluator` | Publishes this reference page. |
-| Skills | `nemo.skills:evaluator` | Publishes the evaluator plugin development skill. |
+| Surface | Entry point | Behavior |
+| --- | --- | --- |
+| CLI | `nemo.cli:evaluator` | Plugin status, metric discovery, job schema inspection, local runs, and durable submissions |
+| Service | `nemo.services:evaluator` | Health, job, stored-resource, and result routes |
+| SDK | `nemo.sdk:evaluator` | `client.evaluator` execution, job lifecycle, stored resources, and result indexes |
+| Dataset job | `nemo.jobs:evaluator.evaluate` | Scores inline or Fileset-backed datasets |
+| Agent job | `nemo.jobs:evaluator.agent-evaluate` | Runs or rescores task-driven agent trials |
+| Docs | `nemo.docs:evaluator` | Publishes this reference |
+| Skill | `nemo.skills:evaluator` | Publishes the evaluator agent skill |
-## Current Job
+Confirm the installed surfaces:
-`evaluator.evaluate` is a `NemoJob` that calls `packages/nemo_evaluator_sdk.Evaluator` directly. It currently supports inline datasets with `exact-match` and `string-check` metric configs.
+```bash
+uv run nemo evaluator info
+uv run nemo evaluator evaluate explain
+uv run nemo evaluator agent-evaluate explain
+```
+When the plugin is installed outside this repository, omit the `uv run` prefix.
-## CLI Examples
+## Dataset-driven evaluation
-### Prerequisite for online evaluation and model-backed metrics
+`evaluator.evaluate` accepts:
-#### Set API key
+- `metrics`: one or more inline metric bundles or stored metric references.
+- `dataset`: inline rows or a `FilesetRef`.
+- `params`: offline, online, or online-model run configuration.
+- `target`: an optional `Model` or `Agent`.
+- `prompt_template`: the online generation prompt.
+- `field_mapping`: canonical evaluator fields mapped to dataset columns.
-Online evaluation examples call [NVIDIA-hosted models](https://build.nvidia.com/models) through the API key referenced by each spec's `api_key_secret`.
+Run the checked offline pass/fail example:
-To generate an API key on the NVIDIA Build hub:
+```bash
+uv run nemo evaluator evaluate run \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
+```
-1. Sign in to your NVIDIA account at .
-2. Open [API Keys](https://build.nvidia.com/settings/api-keys) and click **Generate API Key**.
-3. Export the key before running the CLI: `export NVIDIA_API_KEY=`.
+Submit the same shape as a durable job:
-#### How to use API key
+```bash
+uv run nemo evaluator evaluate submit \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
+```
-For evaluator API key auth, see [Evaluator API Auth](../../../../../skills/nemo-evaluator-plugin/references/api-auth.md)
+The checked LLM-judge example is
+`skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json`.
-### Examples
+### Python SDK
-Check that the plugin is installed and reports the registered job key:
+`client.evaluator.run` executes one runtime metric locally and returns an
+`EvaluationResult`. `submit` creates a durable job resource:
-```bash
-nemo evaluator info
-```
+```python
+from nemo_evaluator_sdk import ExactMatchMetric
+from nemo_platform import NeMoPlatform
+
+client = NeMoPlatform(base_url="http://localhost:8080", workspace="default")
+metric = ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+)
+dataset = [
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+]
-Inspect the generated job metadata:
+local_result = client.evaluator.run(metric=metric, dataset=dataset)
-```bash
-nemo evaluator evaluate explain
+job = client.evaluator.submit(metric=metric, dataset=dataset)
+job.wait_until_done()
+remote_result = job.get_result()
+artifact_dir = job.download_artifacts("evaluation-artifacts")
```
-Run an inline exact-match metric:
+Always wait for terminal completion before retrieving results. Metric progress
+can reach 100 percent before the platform finalizes result artifacts.
-```bash
-nemo evaluator evaluate run --spec '{"metric":{"type":"exact-match","reference":"{{item.expected}}","candidate":"{{item.model_output}}"},"dataset":[{"expected":"blue","model_output":"Blue"},{"expected":"Jupiter","model_output":"Saturn"}],"params":{"parallelism":2}}'
-```
+Local dataset inputs may also be paths. Local and durable jobs accept
+`FilesetRef`. Durable submission additionally accepts a `ModelRef`; local
+execution requires a concrete `Model`.
-Run an online llm-as-judge metric from a spec file (requires `NVIDIA_API_KEY`, see the [prerequisite](#prerequisite-for-online-evaluation-and-model-backed-metrics) above):
+## Task-driven agent evaluation
-```bash
-nemo evaluator evaluate run --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/llm_as_judge.json
-```
+`evaluator.agent-evaluate` accepts inline tasks or a stored `TasksetRef`. Each
+task has its own metrics. Provide exactly one of:
-Run a benchmark metric from spec file example:
+- `target` to generate trials.
+- `trials` to rescore precomputed trials.
-```bash
-nemo evaluator evaluate run --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_benchmark.json
-```
+Durable target variants are:
-## Python Examples
+- `ModelTarget`
+- `AgentTarget`
+- `CodexRunnerTarget`
+- `FabricRunnerTarget`
+- `HarborRunnerTarget`
-Read the plugin service status through the platform SDK namespace:
+The standalone SDK additionally accepts a direct `AgentTaskRunner`:
```python
-from nemo_platform import NeMoPlatform
-
-client = NeMoPlatform(base_url="http://localhost:8080")
-status = client.evaluator.plugin_status()
+result = await AgentEvaluator().run(tasks=tasks, target=runner)
```
-Use the evaluator SDK directly, matching the job's current execution path:
+Use `max_concurrent_tasks` for task-level concurrency. Target-specific fields
+control inference or runner concurrency. `fail_fast` stops on the first scoring
+failure, and `benchmark` records run metadata.
-```python
-from nemo_evaluator_sdk import Evaluator
-from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+Harbor tasks must preserve `harbor_dataset_path` metadata from task discovery.
+The runtime also requires Harbor, Docker, and any selected agent dependencies.
-metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}")
-result = Evaluator().run_sync(
- metrics=metric,
- dataset=[{"expected": "blue", "model_output": "Blue"}],
-)
-```
+Agent-evaluation jobs write trials, scores, summaries, evidence, and report
+artifacts, and create a queryable record under
+`client.evaluator.agent_eval_results`.
+
+## Stored resources
+
+`client.evaluator` exposes immutable definitions and persisted result indexes:
+
+| Resource | Operations | Notable filters |
+| --- | --- | --- |
+| `metrics` | create, retrieve, list, delete | `metric_type`, `include_derived` |
+| `tasks` | create, retrieve, list, delete | pagination and sort |
+| `tasksets` | create, retrieve, list, delete | pagination and sort |
+| `eval_results` | retrieve, list, delete | `job_id`, target, `dataset_ref` |
+| `agent_eval_results` | retrieve, list, delete | `job_id`, target |
+
+Metrics, tasks, and tasksets have no update operation. Delete and recreate them
+or use a new versioned name.
+
+Stored tasks keep metric references but do not keep grader-only task
+`reference`; use inline agent-evaluation tasks when a metric needs held-out
+per-task data.
+
+## Authentication and metric packaging
+
+For local execution, `api_key_secret` names a local environment variable. For
+durable submission, it names a NeMo Platform secret in the target workspace.
+Inspect a remote 409 response body because it can report a missing platform
+secret rather than a duplicate job.
+
+Built-in metrics use declarative inline bundles by default. Submitting custom
+Python metrics requires an explicit `HybridMetricBundlePackager` or
+`CloudpickleMetricBundlePackager`.
+
+See the canonical
+[Evaluator plugin skill](../../../../../skills/nemo-evaluator-plugin/SKILL.md)
+for the execution workflow and focused references.
diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py
index 8c03b81a34..36b7f0260a 100644
--- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py
+++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import json
from collections.abc import Sequence
from pathlib import Path
from typing import Any, cast
@@ -31,7 +32,7 @@
Target,
)
from nemo_evaluator.metric_refs import MetricRef
-from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric
+from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric
from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager
from nemo_evaluator.tasks.agent_evaluate import main as agent_eval_task_main
from nemo_evaluator.tasks.runner import SDK_INITIALIZATION_EXIT_CODE
@@ -66,6 +67,10 @@ def _inline_metric() -> MetricInline:
return MetricInline.model_validate(bundle.model_dump(mode="json"))
+def _repo_root() -> Path:
+ return Path(__file__).resolve().parents[3]
+
+
def _task_spec() -> AgentEvalTaskSpec:
return AgentEvalTaskSpec(
id="task-1",
@@ -436,6 +441,38 @@ def _assert_agent_eval_step_entrypoint(job_spec: PlatformJobSpec) -> None:
assert container.command == ["nemo_evaluator.tasks.agent_evaluate"]
+async def test_checked_fabric_spec_transforms_and_compiles() -> None:
+ path = _repo_root() / "skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json"
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ input_spec = AgentEvalInputSpec.model_validate(payload)
+
+ spec = await AgentEvalJob.to_spec(
+ input_spec,
+ workspace="default",
+ entity_client=None,
+ async_sdk=None,
+ is_local=False,
+ )
+
+ assert isinstance(spec, AgentEvalSpec)
+ assert isinstance(spec.tasks, list)
+ bundle = MetricBundle.model_validate(spec.tasks[0].metrics[0].model_dump(mode="json"))
+ assert bundle.payload.kind == "inline"
+
+ compiled = await AgentEvalJob.compile(
+ workspace="default",
+ spec=spec,
+ entity_client=None,
+ job_name=None,
+ async_sdk=None,
+ )
+ job_spec = PlatformJobSpec.model_validate(compiled)
+ _assert_agent_eval_step_entrypoint(job_spec)
+ config = cast(dict[str, Any], job_spec.steps[0].config)
+ assert config["target"]["kind"] == "fabric"
+ assert config["tasks"][0]["metrics"][0]["payload"]["kind"] == "inline"
+
+
@pytest.mark.parametrize(
("target", "expected_kind", "expected_endpoint_name"),
[
diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py
index 147fbffd83..b37a5bfde8 100644
--- a/plugins/nemo-evaluator/tests/test_evaluate_job.py
+++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py
@@ -6,7 +6,6 @@
from __future__ import annotations
import json
-from collections.abc import Callable
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Literal, cast
@@ -16,7 +15,6 @@
import pytest
from nemo_evaluator.cli import EvaluatorPluginCLI
from nemo_evaluator.filesets import FilesetRef
-from nemo_evaluator.jobs.compiler import compile_evaluate_job
from nemo_evaluator.jobs.evaluate import (
AGGREGATE_SCORES_RESULT_NAME,
ARTIFACTS_RESULT_NAME,
@@ -27,6 +25,7 @@
EvaluateJob,
EvaluateSpec,
)
+from nemo_evaluator.jobs.metric_resolution import to_runtime_bundle
from nemo_evaluator.resolvers import PlatformModelResolver, _parse_required_workspace_name
from nemo_evaluator.shared.metric_bundles.bundles import (
MetricBundle,
@@ -44,7 +43,6 @@
from nemo_evaluator_sdk.metrics.f1 import F1Metric
from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric
from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult
-from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric
from nemo_evaluator_sdk.values import (
Agent,
AggregatedMetricResult,
@@ -69,9 +67,7 @@
from pytest_mock import MockerFixture
from typer.testing import CliRunner
-ExampleSpecBuilder = Callable[[], dict[str, Any]]
EXAMPLE_SPEC_PATHS = (
- Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json"),
Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json"),
Path("skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json"),
)
@@ -98,152 +94,6 @@ def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]
-def _generated_exact_match_metric_spec() -> dict[str, Any]:
- return {
- "metrics": [
- _bundle_payload(ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}")),
- ],
- "dataset": [
- {"expected": "blue", "model_output": "Blue"},
- {"expected": "Jupiter", "model_output": "Saturn"},
- ],
- "params": {"parallelism": 2},
- }
-
-
-def _generated_exact_match_benchmark_spec() -> dict[str, Any]:
- return {
- "metrics": [
- _bundle_payload(ExactMatchMetric(reference="{{item.reference}}")),
- _bundle_payload(
- StringCheckMetric(
- operation="contains",
- left_template="{{sample.output_text}}",
- right_template="{{item.required_phrase}}",
- )
- ),
- ],
- "dataset": [
- {
- "prompt": "Return exactly this word with no punctuation: Paris",
- "reference": "Paris",
- "required_phrase": "Paris",
- },
- {
- "note": (
- "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are "
- "'London' so both metrics should report a miss."
- ),
- "prompt": "Return exactly this word with no punctuation: Oslo",
- "reference": "London",
- "required_phrase": "London",
- },
- ],
- "params": {
- "parallelism": 4,
- "limit_samples": 2,
- "ignore_request_failure": False,
- "request_timeout": 60,
- "max_retries": 3,
- },
- "target": {
- "url": "https://integrate.api.nvidia.com/v1/chat/completions",
- "name": "nvidia/nemotron-3-super-120b-a12b",
- "api_key_secret": "NVIDIA_API_KEY",
- "format": "nim",
- },
- "prompt_template": {
- "messages": [
- {
- "role": "user",
- "content": "{{item.prompt}}",
- }
- ]
- },
- }
-
-
-def _generated_llm_as_judge_spec() -> dict[str, Any]:
- return {
- "metrics": [
- _bundle_payload(
- LLMJudgeMetric(
- model=Model(
- url="https://integrate.api.nvidia.com/v1/chat/completions",
- name="nvidia/nemotron-3-super-120b-a12b",
- api_key_secret=SecretRef(root="NVIDIA_API_KEY"),
- format="nim",
- ),
- scores=[
- RangeScore(
- name="helpfulness",
- description="How well does the response help the user?",
- minimum=0,
- maximum=4,
- parser=JSONScoreParser(json_path="helpfulness"),
- )
- ],
- prompt_template={
- "messages": [
- {
- "role": "system",
- "content": (
- "You are an evaluator. Rate the response's helpfulness from 0-4. "
- "Return only a JSON object with this shape: "
- '{"helpfulness": }.'
- ),
- },
- {
- "role": "user",
- "content": (
- "User prompt: {{item.input}}\n\n"
- "Assistant response: "
- "{{sample.output_text | default(item.output)}}\n\n"
- "Rate this response."
- ),
- },
- ]
- },
- )
- )
- ],
- "dataset": [
- {"input": "What is the capital of France?"},
- {"input": "How do I make scrambled eggs?"},
- ],
- "params": {
- "parallelism": 2,
- "limit_samples": 2,
- "request_timeout": 120,
- "max_retries": 3,
- },
- "target": {
- "url": "https://integrate.api.nvidia.com/v1/chat/completions",
- "name": "nvidia/nemotron-3-super-120b-a12b",
- "api_key_secret": "NVIDIA_API_KEY",
- "format": "nim",
- },
- "prompt_template": {
- "messages": [
- {
- "role": "user",
- "content": "{{item.input}}",
- }
- ]
- },
- }
-
-
-def _example_spec_builders() -> dict[Path, ExampleSpecBuilder]:
- return {
- Path(
- "skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json"
- ): _generated_exact_match_benchmark_spec,
- Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json"): _generated_exact_match_metric_spec,
- Path("skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json"): _generated_llm_as_judge_spec,
- }
-
-
def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None:
step = job_spec.steps[0]
container = cast(Any, step.executor).container
@@ -414,7 +264,7 @@ def _llm_judge_ref_metric() -> LLMJudgeMetric:
"spec_path",
EXAMPLE_SPEC_PATHS,
)
-def test_checked_in_example_spec_uses_metric_bundle_shape(spec_path: Path) -> None:
+def test_checked_in_example_spec_uses_inline_metric_bundle_shape(spec_path: Path) -> None:
payload = json.loads((_repo_root() / spec_path).read_text(encoding="utf-8"))
spec = EvaluateInputSpec.model_validate(payload)
@@ -423,21 +273,40 @@ def test_checked_in_example_spec_uses_metric_bundle_shape(spec_path: Path) -> No
assert len(spec.metrics) >= 1
for metric_payload in payload["metrics"]:
bundle = MetricBundle.model_validate(metric_payload)
- # Static cloudpickle fixtures are Python-minor-version specific, so
- # this test validates the checked-in bundle envelope without hydrating.
- assert bundle.payload.kind == "cloudpickle"
+ assert bundle.payload.kind == "inline"
+ assert {
+ "python_version",
+ "cloudpickle_version",
+ "pickle_protocol",
+ "blob",
+ }.isdisjoint(metric_payload["payload"])
assert bundle.metric_type == metric_payload["metric_type"]
+ assert unbundle_metric(bundle).type == bundle.metric_type
@pytest.mark.parametrize(
"spec_path",
EXAMPLE_SPEC_PATHS,
)
-def test_generated_example_spec_compiles_with_runtime_cloudpickle(spec_path: Path) -> None:
- payload = _example_spec_builders()[spec_path]()
+async def test_checked_in_example_spec_transforms_and_compiles(spec_path: Path) -> None:
+ payload = json.loads((_repo_root() / spec_path).read_text(encoding="utf-8"))
- spec = EvaluateSpec.model_validate(payload)
- compiled = compile_evaluate_job(spec)
+ input_spec = EvaluateInputSpec.model_validate(payload)
+ spec = await EvaluateJob.to_spec(
+ input_spec,
+ workspace="default",
+ entity_client=None,
+ async_sdk=None,
+ is_local=False,
+ )
+ assert isinstance(spec, EvaluateSpec)
+ compiled = await EvaluateJob.compile(
+ workspace="default",
+ spec=spec,
+ entity_client=None,
+ job_name=None,
+ async_sdk=None,
+ )
assert "metric" not in payload
assert len(spec.metrics) >= 1
@@ -498,7 +367,7 @@ def test_cli_explain_uses_registered_evaluator_job_key() -> None:
assert payload["spec_schema"]["title"] == "EvaluateSpec"
-def test_cli_info_reports_registered_evaluator_job_key() -> None:
+def test_cli_info_reports_registered_evaluator_job_keys() -> None:
app = EvaluatorPluginCLI().get_cli()
add_job_commands(app, {"evaluator.evaluate": EvaluateJob})
@@ -506,7 +375,7 @@ def test_cli_info_reports_registered_evaluator_job_key() -> None:
assert result.exit_code == 0
payload = json.loads(result.output)
- assert payload["jobs"] == ["evaluator.evaluate"]
+ assert payload["jobs"] == ["evaluator.evaluate", "evaluator.agent-evaluate"]
def test_cli_metric_types_reports_sdk_metric_union_types() -> None:
@@ -745,7 +614,7 @@ async def test_evaluate_job_to_spec_resolves_bundled_metric_model_refs_before_co
is_local=False,
)
assert isinstance(canonical, EvaluateSpec)
- canonical_metric = unbundle_metric(canonical.metrics[0])
+ canonical_metric = unbundle_metric(to_runtime_bundle(canonical.metrics[0]))
assert isinstance(canonical_metric, LLMJudgeMetric)
assert isinstance(canonical_metric.model, Model)
assert canonical_metric.model.name == "judge"
@@ -799,7 +668,7 @@ async def test_evaluate_job_to_spec_preserves_metric_without_model_refs() -> Non
)
assert isinstance(canonical, EvaluateSpec)
- metric = unbundle_metric(canonical.metrics[0])
+ metric = unbundle_metric(to_runtime_bundle(canonical.metrics[0]))
assert isinstance(metric, LLMJudgeMetric)
assert metric.prompt_template is None
diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py
new file mode 100644
index 0000000000..6cee080505
--- /dev/null
+++ b/plugins/nemo-evaluator/tests/test_skill_examples.py
@@ -0,0 +1,325 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Focused validation for the canonical Evaluator plugin skill examples."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+from types import ModuleType
+from typing import Any
+
+import pytest
+from nemo_evaluator.api.schemas import TasksetRef
+from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, CodexRunnerTarget, FabricRunnerTarget
+from nemo_evaluator.jobs.evaluate import EvaluateInputSpec
+from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric, unbundle_metric
+from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager
+from nemo_evaluator_sdk import ExactMatchMetric, LLMJudgeMetric, Model
+from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
+from nemo_evaluator_sdk.agent_eval.persistence import read_trials
+from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus
+from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask
+from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput
+
+
+def _repo_root() -> Path:
+ return Path(__file__).resolve().parents[3]
+
+
+def _load_module(relative_path: str, name: str) -> ModuleType:
+ path = _repo_root() / relative_path
+ spec = importlib.util.spec_from_file_location(name, path)
+ if spec is None or spec.loader is None:
+ raise RuntimeError(f"Could not load {path}")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _fenced_block_containing(markdown: str, *, language: str, needle: str) -> str:
+ marker = f"```{language}\n"
+ for remainder in markdown.split(marker)[1:]:
+ source = remainder.split("```", 1)[0]
+ if needle in source:
+ return source
+ raise ValueError(f"no {language} block contains {needle!r}")
+
+
+def test_generated_skill_specs_are_current_and_inline() -> None:
+ generator = _load_module(
+ "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py",
+ "nemo_evaluator_skill_spec_generator",
+ )
+
+ assert generator.check_specs() == 0
+ for payload in generator.generated_specs().values():
+ spec = EvaluateInputSpec.model_validate(payload)
+ assert spec.metrics
+ for metric in spec.metrics:
+ bundle = MetricBundle.model_validate(metric.model_dump(mode="json"))
+ assert bundle.payload.kind == "inline"
+
+
+def test_generated_llm_judge_spec_uses_local_environment_secret() -> None:
+ generator = _load_module(
+ "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py",
+ "nemo_evaluator_skill_secret_generator",
+ )
+
+ payload = generator.build_llm_as_judge_spec()
+ assert payload["target"]["api_key_secret"] == "NVIDIA_API_KEY"
+
+ bundle = MetricBundle.model_validate(payload["metrics"][0])
+ assert bundle.secrets["NVIDIA_API_KEY"].root == "NVIDIA_API_KEY"
+
+ judge = unbundle_metric(bundle)
+ assert isinstance(judge, LLMJudgeMetric)
+ assert isinstance(judge.model, Model)
+ assert judge.model.api_key_secret is not None
+ assert judge.model.api_key_secret.root == "NVIDIA_API_KEY"
+
+
+def test_local_llm_judge_spec_guides_platform_secret_remap() -> None:
+ root = _repo_root() / "skills/nemo-evaluator-plugin/references"
+ auth = (root / "api-auth.md").read_text(encoding="utf-8")
+ execution = (root / "execution.md").read_text(encoding="utf-8")
+ normalized_auth = " ".join(auth.split())
+
+ assert ".target.api_key_secret = $platform_secret" in auth
+ assert ".metrics[0].secrets.NVIDIA_API_KEY = $platform_secret" in auth
+ assert "Do not edit `metrics[*].payload`" in normalized_auth
+ assert "api-auth.md#adapt-the-local-first-spec-for-platform-submission" in execution
+ assert "--spec-file llm_as_judge.platform.json" in execution
+
+
+def test_skill_python_examples_import_and_build_agent_spec() -> None:
+ examples = _load_module(
+ "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py",
+ "nemo_evaluator_skill_examples",
+ )
+ metric_bundle = bundle_metric(
+ examples.capital_france_metric(),
+ InlineMetricBundlePackager(),
+ ).model_dump(mode="json")
+
+ spec = AgentEvalInputSpec.model_validate(examples.build_agent_eval_spec(metric_bundle))
+
+ assert not isinstance(spec.tasks, TasksetRef)
+ assert len(spec.tasks) == 1
+ assert isinstance(spec.target, CodexRunnerTarget)
+
+
+def test_skill_standalone_example_scores_pass_and_failure() -> None:
+ examples = _load_module(
+ "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py",
+ "nemo_evaluator_skill_standalone_example",
+ )
+
+ result = examples.evaluate_standalone()
+
+ assert len(result.row_scores) == 2
+ assert result.aggregate_scores.scores[0].mean == 0.5
+
+
+def test_skill_agent_metric_scores_precomputed_output() -> None:
+ examples = _load_module(
+ "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py",
+ "nemo_evaluator_skill_agent_metric",
+ )
+ task = AgentEvalTask(
+ id="capital-france",
+ intent="Name the capital of France.",
+ inputs={"instruction": "What is the capital of France?"},
+ metrics=[examples.capital_france_metric()],
+ )
+ trial = AgentEvalTrial(
+ id="trial-1",
+ task_id=task.id,
+ status=AgentEvalTrialStatus.COMPLETED,
+ output=AgentOutput(output_text="Paris"),
+ )
+
+ result = AgentEvaluator().run_sync(tasks=[task], trials=[trial])
+
+ assert result.scores[0].status is AgentEvalScoreStatus.COMPLETED
+ assert result.scores[0].outputs[0].value == 1.0
+
+
+def test_checked_durable_fabric_job_is_a_valid_agent_eval_spec() -> None:
+ generator = _load_module(
+ "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py",
+ "nemo_evaluator_skill_agent_spec_generator",
+ )
+ path = _repo_root() / "skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json"
+ generated = generator.generated_agent_specs()
+
+ assert generated[path] == json.loads(path.read_text(encoding="utf-8"))
+ spec = AgentEvalInputSpec.model_validate(generated[path])
+
+ assert isinstance(spec.tasks, list)
+ metric_payload = spec.tasks[0].metrics[0].model_dump(mode="json")
+ bundle = MetricBundle.model_validate(metric_payload)
+ assert bundle.payload.kind == "inline"
+ assert {
+ "python_version",
+ "cloudpickle_version",
+ "pickle_protocol",
+ "blob",
+ }.isdisjoint(metric_payload["payload"])
+ metric = unbundle_metric(bundle)
+ assert isinstance(metric, ExactMatchMetric)
+ assert isinstance(spec.target, FabricRunnerTarget)
+ assert spec.target.capture_trajectory is False
+ assert spec.max_concurrent_tasks == 1
+
+
+def test_readme_standalone_direct_runner_scores_task(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ readme = (_repo_root() / "plugins/nemo-evaluator/README.md").read_text(encoding="utf-8")
+ source = _fenced_block_containing(readme, language="python", needle="CallableAgentTaskRunner(answer)")
+ namespace: dict[str, Any] = {}
+ monkeypatch.chdir(tmp_path)
+
+ exec(compile(source, "plugins/nemo-evaluator/README.md", "exec"), namespace)
+
+ result = namespace["result"]
+ assert result.trials[0].output.output_text == "Paris"
+ assert result.scores[0].outputs[0].value == 1.0
+
+
+def test_skill_points_to_working_repository_fabric_installer() -> None:
+ root = _repo_root()
+ skill = (root / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8")
+
+ assert "uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact" in skill
+ assert "script/dev-install-fabric.sh" in skill
+ assert (root / "script/dev-install-fabric.sh").is_file()
+
+
+def test_skill_explains_cli_discovery_commands() -> None:
+ skill = (_repo_root() / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8")
+
+ assert "# confirms plugin readiness and lists the registered evaluator jobs." in skill
+ assert "# lists available metric names; add a metric name to print its schema." in skill
+ assert "# next two commands print the dataset-driven and task-driven job input and" in skill
+ assert "# output schemas - can be very large" in skill
+
+
+def test_skill_routes_dataset_examples_to_references() -> None:
+ skill = (_repo_root() / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8")
+
+ assert "references/execution.md#validate-standalone-then-submit-to-the-platform" in skill
+ assert "references/execution.md#getting-job-results" in skill
+ assert "references/resources.md#store-a-metric-task-and-taskset" in skill
+ assert "references/resources.md#query-persisted-results" in skill
+ assert "result = Evaluator().run_sync(" not in skill
+ assert "job = client.evaluator.submit(" not in skill
+
+
+def test_skill_links_to_evaluation_shape_guidance() -> None:
+ root = _repo_root() / "skills/nemo-evaluator-plugin"
+ skill = (root / "SKILL.md").read_text(encoding="utf-8")
+ reference = root / "references/evaluation-shapes.md"
+ guidance = reference.read_text(encoding="utf-8")
+
+ assert "references/evaluation-shapes.md#dataset-driven-evaluation" in skill
+ assert "references/evaluation-shapes.md#task-driven-evaluation" in skill
+ assert "[SDK Execution](execution.md)" in guidance
+ assert "[Agent Evaluation](agent-evaluation.md)" in guidance
+
+
+def test_skill_names_concrete_standalone_agent_targets() -> None:
+ root = _repo_root() / "skills/nemo-evaluator-plugin"
+ skill = (root / "SKILL.md").read_text(encoding="utf-8")
+ reference = (root / "references/agent-evaluation.md").read_text(encoding="utf-8")
+ sections = (
+ skill.split("**Standalone SDK evaluation**", 1)[1].split("**Platform job evaluation**", 1)[0],
+ reference.split("The standalone target union is:", 1)[1].split("For a minimal direct runner:", 1)[0],
+ )
+
+ for section in sections:
+ assert "`GenericAgent`" in section
+ assert "AgentTaskRunner" in section
+ assert "`Agent`" not in section
+ assert "NemoAgentToolkitAgent" not in section
+
+
+def test_execution_pairs_python_examples_with_cli_when_supported() -> None:
+ reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/execution.md").read_text(encoding="utf-8")
+
+ python_blocks = reference.count("```python")
+ cli_blocks = reference.count("```bash")
+ assert python_blocks
+ assert cli_blocks >= python_blocks
+
+ submit_block = reference.split("**Platform Python SDK**", 1)[1].split("```python", 1)[1].split("```", 1)[0]
+ assert "metric=ExactMatchMetric(" in submit_block
+ assert "dataset=[" in submit_block
+
+
+def test_metric_selection_points_to_metric_protocol() -> None:
+ reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/metric-selection.md").read_text(
+ encoding="utf-8"
+ )
+
+ assert "nemo_evaluator_sdk.metrics.protocol.Metric" in reference
+ assert "nemo_evaluator_sdk.values.protocol.Metric" not in reference
+
+
+def test_multiple_metric_platform_submission_uses_cli() -> None:
+ reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/execution.md").read_text(encoding="utf-8")
+ section = reference.split("## Multiple metrics", 1)[1].split("## Package metrics safely", 1)[0]
+
+ assert "Python SDK" not in section
+ assert "client.evaluator.submit" not in section
+ assert "nemo evaluator evaluate submit --spec-file multi-metric.json" in section
+
+
+def test_resources_show_inline_task_before_held_out_reference_guidance() -> None:
+ reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/resources.md").read_text(encoding="utf-8")
+
+ example_position = reference.index("inline_task = AgentEvalTaskInput(")
+ guidance_position = reference.index("Stored tasks keep metric references.")
+ assert example_position < guidance_position
+ assert 'reference={"expected": "Paris"}' in reference
+
+
+def test_agent_evaluation_shows_how_to_retrieve_stored_trials() -> None:
+ reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/agent-evaluation.md").read_text(
+ encoding="utf-8"
+ )
+
+ assert 'agent_eval_results.retrieve("")' in reference
+ assert "client.files.download(remote_path=stored.bundle_ref" in reference
+ assert 'read_trials("previous-run")' in reference
+ assert "nemo jobs results download agent-eval-results" in reference
+ assert callable(read_trials)
+
+
+def test_authored_skill_guidance_uses_submit_for_plugin_jobs() -> None:
+ root = _repo_root() / "skills/nemo-evaluator-plugin"
+ authored_paths = [
+ root / "SKILL.md",
+ *sorted((root / "references").glob("*.md")),
+ *sorted((root / "assets/examples").glob("*.py")),
+ ]
+ guidance = "\n".join(path.read_text(encoding="utf-8") for path in authored_paths)
+
+ forbidden = (
+ "client.evaluator.create(",
+ "client.evaluator.run(",
+ "nemo evaluator evaluate run",
+ "nemo evaluator agent-evaluate run",
+ "evaluate run/submit",
+ "agent-evaluate run/submit",
+ "local plugin",
+ )
+ assert not any(term in guidance for term in forbidden)
+
+ assert "Evaluator().run_sync(" in guidance
+ assert "AgentEvaluator().run(" in guidance
+ assert "client.evaluator.submit(" in guidance
+ assert "nemo evaluator evaluate submit" in guidance
+ assert "nemo evaluator agent-evaluate submit" in guidance
diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh
index d560455bcd..5443e74178 100755
--- a/script/dev-install-fabric.sh
+++ b/script/dev-install-fabric.sh
@@ -5,21 +5,25 @@
# Dev-only: install the `nemo-relay` GATEWAY BINARY, the one Fabric eval dependency that cannot come
# from a wheel. It is required for live ATIF trajectory capture on out-of-process harnesses (codex).
#
-# Everything else is in the lock — `uv sync --extra fabric` installs the nemo-fabric SDK, the
-# codex/claude/deepagents adapters, and the nemo-relay Python bindings. The pip `nemo-relay` package
-# is bindings-only (its wheel declares no console script and contains no executable), so the daemon is
-# published solely as a GitHub release asset.
+# Everything else is in the lock —
+# `uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact` installs the nemo-fabric
+# SDK, the codex/claude/hermes adapters, and the nemo-relay Python bindings without removing the
+# existing workspace environment. The pip `nemo-relay` package is bindings-only (its wheel declares
+# no console script and contains no executable), so the daemon is published solely as a GitHub
+# release asset.
#
# The version defaults to the `nemo-relay` bindings installed in the venv, so the daemon and the
# bindings cannot drift apart when the lock moves.
#
# To run against an unreleased Fabric instead of the locked wheels, install the checkout directly:
# uv pip install --python .venv/bin/python "/path/to/NeMo-Fabric[codex,relay,runtime]"
-# and `uv sync --extra fabric` to get back to the locked state. (That needs cargo — Fabric builds a
-# Rust/pyo3 extension from source.)
+# and use the package-scoped command above to restore the locked project dependencies without
+# pruning unrelated installed packages. (That needs cargo — Fabric builds a Rust/pyo3 extension
+# from source.)
#
# A live codex run additionally needs the `codex` CLI + `codex login` auth.
-# See plugins/nemo-evaluator/docs/design/fabric-runner-integration.md.
+# See skills/nemo-evaluator-plugin/references/agent-evaluation.md and
+# packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py.
#
# Usage:
# script/dev-install-fabric.sh # version matching the installed bindings
@@ -38,7 +42,8 @@ if [ -z "${NEMO_RELAY_VERSION:-}" ]; then
bindings_version="$("$VENV_PY" -c 'import importlib.metadata as m; print(m.version("nemo-relay"))' 2>/dev/null || true)"
if [ -z "$bindings_version" ]; then
echo "nemo-relay is not installed in $VENV_PY, so the gateway version cannot be derived." >&2
- echo "Run 'uv sync --extra fabric' first, or pass NEMO_RELAY_VERSION= explicitly." >&2
+ echo "Run 'uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact' first," >&2
+ echo "or pass NEMO_RELAY_VERSION= explicitly." >&2
exit 1
fi
NEMO_RELAY_VERSION="$(printf '%s' "$bindings_version" | sed -E 's/([0-9])(a|b|rc)\.?([0-9]+)$/\1-\2.\3/')"
diff --git a/skills/nemo-evaluator-plugin/SKILL.md b/skills/nemo-evaluator-plugin/SKILL.md
index e9966e7927..27ae26ae52 100644
--- a/skills/nemo-evaluator-plugin/SKILL.md
+++ b/skills/nemo-evaluator-plugin/SKILL.md
@@ -1,145 +1,129 @@
---
name: nemo-evaluator-plugin
-description: Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, metric types, or plugin-owned Evaluator skills.
-metadata:
- owner: nemo-platform
- maturity: active
-license: Apache-2.0
+description: Use when evaluating models or agents with the NeMo Evaluator plugin; choosing deterministic, LLM, RAG, or agentic metrics; submitting evaluator jobs; managing stored metrics, tasks, tasksets, or results; or configuring agent-evaluate agent, Codex, Fabric, Harbor, and custom runner targets.
---
# Evaluator Plugin
-Use this skill for evaluation tasks against a running NeMo Platform server. The plugin-backed CLI interface is `nemo evaluator`; the legacy generated `nemo evaluation` API command group is not the target surface for new guidance.
+The Plugin CLI entrypoint is `nemo evaluator`.
-## CLI Interface
-
-### Prerequisites
-
-- all commands in this file assume that the shell's working dir is at the root of the Nvidia-NeMo/nemo-platform repo
-- activate the Python virtual environment before invoking the `nemo` CLI: `source .venv/bin/activate`
-
-Check plugin status from the CLI:
-
-```bash
-nemo evaluator info
-```
-
-## Metric Types
-
-### Explore Available Metrics
+## Follow the evaluation loop
-To view available metric names, run:
+1. Clarify whether the input is [dataset-driven rows](references/evaluation-shapes.md#dataset-driven-evaluation)
+ or [task-driven agent work](references/evaluation-shapes.md#task-driven-evaluation).
+2. Choose the simplest metric that measures the requested behavior. Prefer deterministic metrics when possible.
+3. Build a tiny smoke case with one expected pass and one expected failure.
+4. Validate metric behavior with the standalone SDK and inspect row-level output plus aggregates.
+5. Fix field mappings, prompts, parsers, or task definitions before scaling.
+6. Submit the platform job only after the input and scoring shape works.
-```bash
-nemo evaluator metric-types
-```
-
-To view a specific metric schema, pass a metric name from the `metric_types` list above:
-
-```bash
-nemo evaluator metric-types
-```
-
-Inspect all the registered metric schema contracts:
-
-```bash
-nemo evaluator evaluate explain
-```
+Read [Metric Selection](references/metric-selection.md) before choosing a
+metric for a rubric, RAG workflow, or tool-calling evaluation.
-> Note: use `nemo evaluator evaluate explain` as the source of truth for the current plugin input schema. It will return a large json schema response, so strongly prefer `nemo evaluator metric-types` when you only need metric names and corresponding schemas.
+## Choose the execution interface
-## Evaluation Spec
+| Need | Interface |
+| --- | --- |
+| Fast metric iteration without NeMo Platform | `nemo_evaluator_sdk.Evaluator` |
+| Dataset-driven platform job | `client.evaluator.submit(...)` or `nemo evaluator evaluate submit` |
+| Multiple inline/stored metric refs in one job | `nemo evaluator evaluate submit` with an `EvaluateInputSpec` |
+| Task-driven platform job | `nemo evaluator agent-evaluate submit` |
+| Reusable platform definitions and result indexes | `client.evaluator.metrics`, `.tasks`, `.tasksets`, `.eval_results`, `.agent_eval_results` |
-Evaluation spec is a payload that is provided to CLI as an input to execute evaluation.
+- Read [SDK Execution](references/execution.md) for datasets, targets,
+configuration, field mapping, job lifecycle, and custom metric packaging.
+- Read [Stored Resources](references/resources.md) for persisted definitions and
+result queries.
-At a high level, a spec describes:
-
-- `metrics`: bundled Evaluator SDK metric configurations
-- `dataset`: inline rows to evaluate or platform FilesetRef that contains the dataset
-- `params`: optional Evaluator SDK execution parameters
-- `target`: optional model or agent target for online evaluation
-
-See the LLM-judge spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json).
+## CLI Interface
-### Metric Bundle Payloads
+### Prerequisites
-The checked-in [spec examples](./assets/specs) use bundled SDK metrics. The fields under `metrics[*].payload` are generated by `bundle_metric(metric, CloudpickleMetricBundlePackager())`.
+All commands in this file assume that the shell's working directory is the root
+of the NVIDIA-NeMo/nemo-platform repository.
-To see the pattern for configuring a pre-defined SDK metric, for example `ExactMatchMetric`, and converting it into bundled metric JSON, inspect `build_metric_bundle_example()` in [generate_example_specs.py](./scripts/generate_example_specs.py) and run:
+In a NeMo Platform repository checkout, run commands through the workspace:
```bash
-uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py
+# confirms plugin readiness and lists the registered evaluator jobs.
+uv run nemo evaluator info
+# lists available metric names; add a metric name to print its schema.
+uv run nemo evaluator metric-types
+# next two commands print the dataset-driven and task-driven job input and
+# output schemas - can be very large, use with caution to avoid filling up the context window.
+uv run nemo evaluator evaluate explain
+uv run nemo evaluator agent-evaluate explain
```
-## Run Evaluations
+When the skill and plugin are installed, use the installed `nemo` command
+without assuming a repository root or manually activating `.venv`.
-### Run Using File Spec Reference
+Resolve bundled assets relative to this skill directory. In this repository the
+canonical path is `skills/nemo-evaluator-plugin`; an installed skill may live
+under a different skills root.
-When using the `nemo evaluator evaluate run` command, results are saved into local temporary directories and the link is printed to stdout.
-Prefer the `--spec-file` named argument over inline shell JSON because metric bundles include serialized payloads.
-Examples of various specs are provided in the [assets/specs](./assets/specs/) directory.
+## Dataset-driven evaluation examples
-#### Evaluate using `exact-match` metric
+- Follow [Validate standalone, then submit to the platform](references/execution.md#validate-standalone-then-submit-to-the-platform)
+ for the two-row pass/fail smoke test and its CLI submission.
+- Follow [Map noncanonical fields](references/execution.md#map-noncanonical-fields)
+ when dataset columns need `field_mapping`.
+- Follow [Getting job results](references/execution.md#getting-job-results)
+ for submission, terminal waiting, result retrieval, and artifact download.
+- Follow [Store a metric, task, and taskset](references/resources.md#store-a-metric-task-and-taskset)
+ for reusable definitions, and [Query persisted results](references/resources.md#query-persisted-results)
+ for result lookup.
-See the spec example at [assets/specs/exact_match_metric.json](./assets/specs/exact_match_metric.json).
+Always call `job.wait_until_done()` before retrieving results or downloading
+artifacts; metric progress can reach 100 percent before the platform job is
+terminal.
-```bash
-nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
-```
+## Task-driven agent evaluation examples
-#### Evaluate using a benchmark metric set
+**Standalone SDK evaluation**
-```bash
-nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json
-```
+Use `AgentEvaluator().run(...)` for standalone task-driven SDK evaluation. Its
+`target` can be a `Model`, a `GenericAgent`, or a direct `AgentTaskRunner`.
-#### Evaluate using `LLM-Judge` metric
+**Platform job evaluation**
-Uses an LLM to score responses. See the spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json).
+Use the plugin `agent-evaluate submit` job for platform task evaluation. Its
+target is a `ModelTarget`, `AgentTarget`, `CodexRunnerTarget`,
+`FabricRunnerTarget`, or `HarborRunnerTarget`; alternatively provide
+precomputed `trials`. Provide exactly one of `target` or `trials`.
-```bash
-nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json
-```
+Submission accepts inline tasks or a stored `TasksetRef`. Stored tasksets are
+resolved in the target workspace.
-### Run Evaluation As A Durable Job
+Read [Agent Evaluation](references/agent-evaluation.md) for inline tasks,
+`TasksetRef`, concurrency, fail-fast behavior, result artifacts, and runner
+configuration.
-Use the `nemo evaluator evaluate submit` command to create a durable evaluation job. The response of this command returns a job handler object instead of the evaluation result.
+### Prepare Fabric in a repository checkout
-```bash
-nemo evaluator evaluate submit \
- --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
-```
-
-The submit response includes the generated job's `name` field, for example `nemo-evaluator-zlhn1ecd`. Wait for the job to complete, then list and download the job results.
+Fabric runner examples and tests need the optional harness adapters and the
+matching Relay gateway:
```bash
-nemo jobs get-status
-nemo jobs get
-nemo jobs results list
-nemo jobs results download aggregate-scores --job --output-file aggregate-scores.json
-nemo jobs results download row-scores --job --output-file row-scores.jsonl
+uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact
+script/dev-install-fabric.sh
```
-## Python SDK Interface
-
-Evaluator Python SDK client is exposed as `evaluator` variable on `NeMoPlatform` instance:
-
-```python
-from nemo_platform import NeMoPlatform
-
-platform_client = NeMoPlatform(base_url="http://localhost:8080")
-status = platform_client.evaluator.plugin_status()
-```
-
-See examples of using the plugin SDK interface in [plugin_sdk_examples.py](./assets/examples/plugin_sdk_examples.py).
-
-## Security
-Make sure not to print any secrets to stdout since this can be collected as logs
+The install script downloads the checksum-verified `nemo-relay` binary that
+matches the locked Python bindings. Add its reported directory to `PATH`, then
+use `uv run --frozen --no-sync ...` for Fabric checks so uv does not remove the
+optional adapters.
-## Additional Resources
+## Read specialized references
-For LLM-judge setup notes, see [LLM Judge Notes](references/llm-judge.md).
+- Read [Evaluator API Auth](references/api-auth.md) before using a model,
+ agent, remote metric, or durable submission.
+- Read [LLM Judge](references/llm-judge.md) before writing judge scores,
+ prompts, or parsers.
+- Read [Troubleshooting](references/troubleshooting.md) when schema,
+ authentication, job, result, or runner behavior fails.
-For evaluator API key auth, see [Evaluator API Auth](references/api-auth.md).
+## Follow security best practices
-For local and cluster troubleshooting, see [Evaluation Troubleshooting](references/troubleshooting.md).
+Never print, serialize, or commit secret values. Store only environment-variable
+names or platform secret references in specs and examples.
diff --git a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py
index e572ae5bb6..1634f9bc27 100644
--- a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py
+++ b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py
@@ -1,109 +1,107 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Local-only Evaluator plugin SDK smoke example.
+"""Concise examples for the Evaluator plugin SDK surfaces.
-The default entrypoint prints an exact-match spec and does not submit jobs or
-call hosted models. Pass --run to execute the same offline metric against a
-running local NeMo Platform.
+These functions are intentionally not called at import time. Copy the one that
+matches the feature being used and supply a configured NeMo Platform client.
"""
from __future__ import annotations
-import argparse
-import gzip
-import json
-import os
-from collections.abc import Iterable
from pathlib import Path
-from tempfile import TemporaryDirectory
from typing import Any
-DEFAULT_BASE_URL = "http://localhost:8080"
-DEFAULT_ROWS = (
- {"expected": "blue", "model_output": "blue"},
- {"expected": "Jupiter", "model_output": "Saturn"},
-)
+def capital_france_metric() -> Any:
+ """Return an output-only metric suitable for stored agent-evaluation tasks."""
+ from nemo_evaluator_sdk import StringCheckMetric
-def write_jsonl_dataset(path: Path, rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> Path:
- """Write rows as JSONL and return the written path."""
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
- return path
-
+ return StringCheckMetric(
+ operation="equals",
+ left_template="{{sample.output_text | trim}}",
+ right_template="Paris",
+ )
-def load_jsonl_rows(path: Path, *, limit: int | None = None) -> list[dict[str, Any]]:
- """Load plain JSONL or .gz JSONL rows."""
- opener = gzip.open if path.suffix == ".gz" else open
- rows: list[dict[str, Any]] = []
- with opener(path, "rt", encoding="utf-8") as stream:
- for line in stream:
- if line.strip():
- rows.append(json.loads(line))
- if limit is not None and len(rows) >= limit:
- break
+def evaluate_standalone() -> Any:
+ """Evaluate one deterministic metric in process."""
+ from nemo_evaluator_sdk import Evaluator, ExactMatchMetric
- return rows
+ return Evaluator().run_sync(
+ metrics=ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+ ),
+ dataset=[
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+ ],
+ )
-def build_exact_match_spec(rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> dict[str, Any]:
- """Build a local exact-match spec that does not require model credentials."""
- from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric
- from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager
- from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
+def submit_and_collect(client: Any, output_dir: Path) -> tuple[Any, Path]:
+ """Submit one metric, wait for completion, and retrieve its artifacts."""
+ from nemo_evaluator_sdk import ExactMatchMetric
- metric = ExactMatchMetric(
- reference="{{item.expected}}",
- candidate="{{item.model_output}}",
+ job = client.evaluator.submit(
+ metric=ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+ ),
+ dataset=[{"expected": "Paris", "output": "Paris"}],
)
- return {
- "metrics": [bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json")],
- "dataset": list(rows),
- "params": {"parallelism": 2, "limit_samples": 2},
- }
-
-
-def run_local_exact_match(dataset_path: Path) -> Any:
- """Run the offline exact-match metric against a local platform."""
- from nemo_evaluator.sdk.types import RunConfig
- from nemo_evaluator_sdk.enums import MetricType
- from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
- from nemo_platform import NeMoPlatform
-
- client = NeMoPlatform(
- base_url=os.environ.get("NMP_BASE_URL", DEFAULT_BASE_URL),
- workspace="default",
+ job.wait_until_done()
+ return job.get_result(), job.download_artifacts(output_dir)
+
+
+def store_resources(client: Any) -> None:
+ """Store one metric, task, and taskset."""
+ from nemo_evaluator.api.schemas import (
+ MetricRef,
+ TaskInput,
+ TaskInputs,
+ TaskRef,
+ TasksetInput,
)
- try:
- evaluator = client.evaluator
- metric = ExactMatchMetric(
- type=MetricType.EXACT_MATCH,
- reference="{{item.expected}}",
- candidate="{{item.model_output}}",
- )
- return evaluator.run(metric=metric, dataset=dataset_path, config=RunConfig(limit_samples=2))
- finally:
- client.close()
-
-def main(argv: list[str] | None = None) -> int:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--run", action="store_true", help="Run local offline exact-match against NeMo Platform.")
- args = parser.parse_args(argv)
-
- with TemporaryDirectory(prefix="nemo-evaluator-smoke-") as tmpdir:
- dataset_path = write_jsonl_dataset(Path(tmpdir) / "exact-match.jsonl")
-
- if args.run:
- result = run_local_exact_match(dataset_path)
- result.print_summary()
- return 0
+ client.evaluator.metrics.create(
+ "answer-exact",
+ metric=capital_france_metric(),
+ )
+ client.evaluator.tasks.create(
+ "capital-france",
+ task=TaskInput(
+ intent="Name the capital of France.",
+ inputs=TaskInputs(instruction="What is the capital of France?"),
+ metrics=[MetricRef("default/answer-exact")],
+ ),
+ )
+ client.evaluator.tasksets.create(
+ "geography",
+ taskset=TasksetInput(tasks=[TaskRef("default/capital-france")]),
+ )
- print(json.dumps(build_exact_match_spec(load_jsonl_rows(dataset_path)), indent=2))
- return 0
+def build_agent_eval_spec(metric_bundle: Any) -> Any:
+ """Build a durable task evaluation with a runner target."""
+ from nemo_evaluator.api.schemas import TaskInputs
+ from nemo_evaluator.jobs.agent_spec import (
+ AgentEvalInputSpec,
+ AgentEvalTaskInput,
+ CodexRunnerTarget,
+ )
-if __name__ == "__main__":
- raise SystemExit(main())
+ return AgentEvalInputSpec(
+ tasks=[
+ AgentEvalTaskInput(
+ id="capital-france",
+ intent="Name the capital of France.",
+ inputs=TaskInputs(instruction="What is the capital of France?"),
+ metrics=[metric_bundle],
+ )
+ ],
+ target=CodexRunnerTarget(model=""),
+ max_concurrent_tasks=2,
+ benchmark={"name": "geography-smoke"},
+ )
diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json
deleted file mode 100644
index 6cf51562c0..0000000000
--- a/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "metrics": [
- {
- "bundle_kind": "metric-bundle",
- "bundle_format_version": "v1",
- "metric_type": "exact-match",
- "metadata": {
- "description": null,
- "labels": {}
- },
- "outputs": [
- {
- "name": "exact-match",
- "description": null,
- "value_json_schema": {
- "description": "Continuous numeric metric value.",
- "title": "ContinuousScore",
- "type": "number"
- }
- }
- ],
- "secrets": {},
- "payload": {
- "python_version": "3.11.15",
- "cloudpickle_version": "3.1.2",
- "pickle_protocol": 5,
- "blob": "gAWVoQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwSe3tpdGVtLnJlZmVyZW5jZX19lIwJY2FuZGlkYXRllE51jBJfX3B5ZGFudGljX2V4dHJhX1-UTowXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QoaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=",
- "digest": "b6d94b6d5a4f304964652358cd55e8e1216664934a9712ac147d566b65ed3b5d",
- "kind": "cloudpickle"
- }
- },
- {
- "bundle_kind": "metric-bundle",
- "bundle_format_version": "v1",
- "metric_type": "string-check",
- "metadata": {
- "description": null,
- "labels": {}
- },
- "outputs": [
- {
- "name": "string-check",
- "description": null,
- "value_json_schema": {
- "description": "Continuous numeric metric value.",
- "title": "ContinuousScore",
- "type": "number"
- }
- }
- ],
- "secrets": {},
- "payload": {
- "python_version": "3.11.15",
- "cloudpickle_version": "3.1.2",
- "pickle_protocol": 5,
- "blob": "gAWV5gEAAAAAAACMJ25lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLnN0cmluZ19jaGVja5SMEVN0cmluZ0NoZWNrTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAxzdHJpbmctY2hlY2uUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJb3BlcmF0aW9ulIwIY29udGFpbnOUjA1sZWZ0X3RlbXBsYXRllIwWe3tzYW1wbGUub3V0cHV0X3RleHR9fZSMDnJpZ2h0X3RlbXBsYXRllIwYe3tpdGVtLnJlcXVpcmVkX3BocmFzZX19lHWMEl9fcHlkYW50aWNfZXh0cmFfX5ROjBdfX3B5ZGFudGljX2ZpZWxkc19zZXRfX5SPlChoHmggaAdoHJCMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=",
- "digest": "5c7c5b74de79b3d84fdd7db4f52393633dee4370036266d088518b94a92b081b",
- "kind": "cloudpickle"
- }
- }
- ],
- "dataset": [
- {
- "prompt": "Return exactly this word with no punctuation: Paris",
- "reference": "Paris",
- "required_phrase": "Paris"
- },
- {
- "note": "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are 'London' so both metrics should report a miss.",
- "prompt": "Return exactly this word with no punctuation: Oslo",
- "reference": "London",
- "required_phrase": "London"
- }
- ],
- "params": {
- "parallelism": 4,
- "limit_samples": 2,
- "ignore_request_failure": false,
- "request_timeout": 60,
- "max_retries": 3
- },
- "target": {
- "url": "https://integrate.api.nvidia.com/v1/chat/completions",
- "name": "nvidia/nemotron-3-super-120b-a12b",
- "api_key_secret": "NVIDIA_API_KEY",
- "format": "nim"
- },
- "prompt_template": {
- "messages": [
- {
- "role": "user",
- "content": "{{item.prompt}}"
- }
- ]
- }
-}
diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
index 4eaee1a54b..c697c588e9 100644
--- a/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
+++ b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
@@ -1,46 +1,53 @@
{
- "metrics": [
+ "metrics": [
+ {
+ "bundle_kind": "metric-bundle",
+ "bundle_format_version": "v1",
+ "metric_type": "exact-match",
+ "metadata": {
+ "description": null,
+ "labels": {}
+ },
+ "outputs": [
{
- "bundle_kind": "metric-bundle",
- "bundle_format_version": "v1",
- "metric_type": "exact-match",
- "metadata": {
- "description": null,
- "labels": {}
- },
- "outputs": [
- {
- "name": "exact-match",
- "description": null,
- "value_json_schema": {
- "description": "Continuous numeric metric value.",
- "title": "ContinuousScore",
- "type": "number"
- }
- }
- ],
- "secrets": {},
- "payload": {
- "python_version": "3.11.15",
- "cloudpickle_version": "3.1.2",
- "pickle_protocol": 5,
- "blob": "gAWVuQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwRe3tpdGVtLmV4cGVjdGVkfX2UjAljYW5kaWRhdGWUjBV7e2l0ZW0ubW9kZWxfb3V0cHV0fX2UdYwSX19weWRhbnRpY19leHRyYV9flE6MF19fcHlkYW50aWNfZmllbGRzX3NldF9flI-UKGgeaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=",
- "digest": "38050e2438a5eef8865ee2ef0bc2ccdaff6991b5d91bd9fb94a8155e759a26a5",
- "kind": "cloudpickle"
- }
+ "name": "exact-match",
+ "description": null,
+ "value_json_schema": {
+ "description": "Continuous numeric metric value.",
+ "title": "ContinuousScore",
+ "type": "number"
+ }
}
- ],
- "dataset": [
- {
- "expected": "blue",
- "model_output": "Blue"
+ ],
+ "secrets": {},
+ "payload": {
+ "metric": {
+ "type": "exact-match",
+ "description": null,
+ "labels": {},
+ "supported_job_types": [
+ "online",
+ "offline"
+ ],
+ "reference": "{{item.expected}}",
+ "candidate": "{{item.output}}"
},
- {
- "expected": "Jupiter",
- "model_output": "Saturn"
- }
- ],
- "params": {
- "parallelism": 2
+ "digest": "bcd49ec7c31e962c06810501e5ea1c67f1667753b65a5a75db10686f6d992ed2",
+ "kind": "inline"
+ }
+ }
+ ],
+ "dataset": [
+ {
+ "expected": "Paris",
+ "output": "Paris"
+ },
+ {
+ "expected": "Paris",
+ "output": "London"
}
+ ],
+ "params": {
+ "parallelism": 2
+ }
}
diff --git a/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json b/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json
new file mode 100644
index 0000000000..3a2d4e7a5f
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json
@@ -0,0 +1,70 @@
+{
+ "tasks": [
+ {
+ "id": "capital-france",
+ "intent": "Name the capital of France.",
+ "inputs": {
+ "instruction": "What is the capital of France?"
+ },
+ "reference": {
+ "expected": "Paris"
+ },
+ "metrics": [
+ {
+ "bundle_kind": "metric-bundle",
+ "bundle_format_version": "v1",
+ "metric_type": "exact-match",
+ "metadata": {
+ "description": null,
+ "labels": {}
+ },
+ "outputs": [
+ {
+ "name": "exact-match",
+ "description": null,
+ "value_json_schema": {
+ "description": "Continuous numeric metric value.",
+ "title": "ContinuousScore",
+ "type": "number"
+ }
+ }
+ ],
+ "secrets": {},
+ "payload": {
+ "metric": {
+ "type": "exact-match",
+ "description": null,
+ "labels": {},
+ "supported_job_types": [
+ "online",
+ "offline"
+ ],
+ "reference": "{{reference.expected}}",
+ "candidate": "{{sample.output_text}}"
+ },
+ "digest": "e48a2f8e509d2ff3fe0b48749baec84577ee590050ff037449ddc5ddcee0045a",
+ "kind": "inline"
+ }
+ }
+ ]
+ }
+ ],
+ "target": {
+ "kind": "fabric",
+ "config": {
+ "metadata": {
+ "name": "readme-fabric-smoke"
+ },
+ "harness": {
+ "adapter_id": "nvidia.fabric.codex"
+ }
+ },
+ "model": "/",
+ "capture_trajectory": false
+ },
+ "max_concurrent_tasks": 1,
+ "fail_fast": true,
+ "benchmark": {
+ "name": "readme-fabric-smoke"
+ }
+}
diff --git a/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json
index 092072c428..7cde5dfb08 100644
--- a/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json
+++ b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json
@@ -1,63 +1,121 @@
{
- "metrics": [
+ "metrics": [
+ {
+ "bundle_kind": "metric-bundle",
+ "bundle_format_version": "v1",
+ "metric_type": "llm-judge",
+ "metadata": {
+ "description": null,
+ "labels": {}
+ },
+ "outputs": [
{
- "bundle_kind": "metric-bundle",
- "bundle_format_version": "v1",
- "metric_type": "llm-judge",
- "metadata": {
- "description": null,
- "labels": {}
- },
- "outputs": [
- {
- "name": "helpfulness",
- "description": "How well does the response help the user?",
- "value_json_schema": {
- "description": "Continuous numeric metric value.",
- "title": "ContinuousScore",
- "type": "number"
- }
- }
- ],
- "secrets": {
- "NVIDIA_API_KEY": "NVIDIA_API_KEY"
- },
- "payload": {
- "python_version": "3.11.15",
- "cloudpickle_version": "3.1.2",
- "pickle_protocol": 5,
- "blob": "gAWVOAkAAAAAAACMJG5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmxsbV9qdWRnZZSMDkxMTUp1ZGdlTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAlsbG0tanVkZ2WUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwFbW9kZWyUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLm1vZGVsc5SMBU1vZGVslJOUKYGUfZQoaAV9lCiMA3VybJSMNGh0dHBzOi8vaW50ZWdyYXRlLmFwaS5udmlkaWEuY29tL3YxL2NoYXQvY29tcGxldGlvbnOUjARuYW1llIwhbnZpZGlhL25lbW90cm9uLTMtc3VwZXItMTIwYi1hMTJilIwPZGVmYXVsdF9oZWFkZXJzlE6MCGhvc3RfdXJslE6MDmFwaV9rZXlfc2VjcmV0lGgTjAlTZWNyZXRSZWaUk5QpgZR9lChoBX2UjARyb290lIwOTlZJRElBX0FQSV9LRVmUc4wXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QojARyb290lJB1YowGZm9ybWF0lGgIjAtNb2RlbEZvcm1hdJSTlIwDbmltlIWUUpR1jBJfX3B5ZGFudGljX2V4dHJhX1-UTmgxj5QoaCloI2glaDSQjBRfX3B5ZGFudGljX3ByaXZhdGVfX5ROdWKMBnNjb3Jlc5RdlIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5zY29yZXOUjApSYW5nZVNjb3JllJOUKYGUfZQoaAV9lChoJYwLaGVscGZ1bG5lc3OUaA6MKUhvdyB3ZWxsIGRvZXMgdGhlIHJlc3BvbnNlIGhlbHAgdGhlIHVzZXI_lIwGcGFyc2VylGg_jA9KU09OU2NvcmVQYXJzZXKUk5QpgZR9lChoBX2UKGgHjARqc29ulIwJanNvbl9wYXRolGhFdWg6Tmgxj5QoaE6QaDxOdWKMB21pbmltdW2USwCMB21heGltdW2USwR1aDpOaDGPlChoR2hRaA5oJWhQkGg8TnViYYwPcHJvbXB0X3RlbXBsYXRllH2UjAhtZXNzYWdlc5RdlCh9lCiMBHJvbGWUjAZzeXN0ZW2UjAdjb250ZW50lIyGWW91IGFyZSBhbiBldmFsdWF0b3IuIFJhdGUgdGhlIHJlc3BvbnNlJ3MgaGVscGZ1bG5lc3MgZnJvbSAwLTQuIFJldHVybiBvbmx5IGEgSlNPTiBvYmplY3Qgd2l0aCB0aGlzIHNoYXBlOiB7ImhlbHBmdWxuZXNzIjogPGludGVnZXI-fS6UdX2UKGhYjAR1c2VylGhajHNVc2VyIHByb21wdDoge3tpdGVtLmlucHV0fX0KCkFzc2lzdGFudCByZXNwb25zZToge3tzYW1wbGUub3V0cHV0X3RleHQgfCBkZWZhdWx0KGl0ZW0ub3V0cHV0KX19CgpSYXRlIHRoaXMgcmVzcG9uc2UulHVlc4wPb3B0aW9uYWxfZmllbGRzlF2UjBFzdHJ1Y3R1cmVkX291dHB1dJR9lIwGc2NoZW1hlH2UKGgHjAZvYmplY3SUjApwcm9wZXJ0aWVzlH2UaEV9lChoB4wHaW50ZWdlcpRoUEsAaFFLBHVzjAhyZXF1aXJlZJRdlGhFYXVzjAlpbmZlcmVuY2WUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLnBhcmFtc5SMD0luZmVyZW5jZVBhcmFtc5STlCmBlH2UKGgFfZQojAt0ZW1wZXJhdHVyZZRHAAAAAAAAAACMCm1heF90b2tlbnOUTQCAjBVtYXhfY29tcGxldGlvbl90b2tlbnOUTowFdG9wX3CUTowEc3RvcJROdWg6fZRoMY-UKGhzaHSQaDxOdWKMDXN5c3RlbV9wcm9tcHSUTowJcmVhc29uaW5nlE6MFmlnbm9yZV9yZXF1ZXN0X2ZhaWx1cmWUiYwIam9iX3R5cGWUaBh1aDpOaDGPlChoX2gcaFNobGg9aGFofJBoPH2UKIwRX3ByZXByb2Nlc3NfaG9va3OUXZQojBxuZW1vX2V2YWx1YXRvcl9zZGsuaW5mZXJlbmNllIwVQWRkSW5mZXJlbmNlUGFyYW1ldGVylJOUKYGUfZSMBnBhcmFtc5R9lChoc0cAAAAAAAAAAGh0TQCAdXNijCRuZW1vX2V2YWx1YXRvcl9zZGsuc3RydWN0dXJlZF9vdXRwdXSUjBlJbmZlcmVuY2VTdHJ1Y3R1cmVkT3V0cHV0lJOUKYGUfZQojAxfanNvbl9zY2hlbWGUfZQoaAdoZWhmaGdoamhrdYwHX3N0cmljdJSJjARtb2RllGiJjBRTdHJ1Y3R1cmVkT3V0cHV0TW9kZZSTlIwRbnZleHRfZ3VpZGVkX2pzb26UhZRSlIwPaW5mZXJlbmNlX3BhcmFtlH2UjApleHRyYV9ib2R5lH2UjAVudmV4dJR9lIwLZ3VpZGVkX2pzb26UaI9zc3N1YmiCjAdMb2dIb29rlJOUKYGUfZSMBmxvZ2dlcpSMB2xvZ2dpbmeUjAlnZXRMb2dnZXKUk5RogoWUUpRzYmWMEl9wb3N0cHJvY2Vzc19ob29rc5RdlGigYYwaX3VzZV9tYXhfY29tcGxldGlvbl90b2tlbnOUiYwIX2FwaV9rZXmUTowHX2NsaWVudJROjA1faW5mZXJlbmNlX2ZulE6MCF9wYXJzZXJzlH2UaEVoP4wPU2NvcmVQYXJzZXJKU09OlJOUKYGUfZQojAVzY29yZZRoQmhOaEVoYWhijAtqc29uX3NjaGVtYZRoZHVic4wMX3Njb3JlX2R1bXBzlH2UaEV9lChoJWhFaA5oRmhQSwBoUUsEdXOMG19wcm9tcHRfdGVtcGxhdGVfaXNfZGVmYXVsdJSJdXViLg==",
- "digest": "dfd6a04359b75b41cba2817bc1496244425e8290102b36c20bd04e7f62b31b8e",
- "kind": "cloudpickle"
- }
+ "name": "helpfulness",
+ "description": "How well the response helps the user.",
+ "value_json_schema": {
+ "description": "Continuous numeric metric value.",
+ "title": "ContinuousScore",
+ "type": "number"
+ }
}
- ],
- "dataset": [
- {
- "input": "What is the capital of France?"
- },
- {
- "input": "How do I make scrambled eggs?"
- }
- ],
- "params": {
- "parallelism": 2,
- "limit_samples": 2,
- "request_timeout": 120,
- "max_retries": 3
- },
- "target": {
- "url": "https://integrate.api.nvidia.com/v1/chat/completions",
- "name": "nvidia/nemotron-3-super-120b-a12b",
- "api_key_secret": "NVIDIA_API_KEY",
- "format": "nim"
- },
- "prompt_template": {
- "messages": [
+ ],
+ "secrets": {
+ "NVIDIA_API_KEY": "NVIDIA_API_KEY"
+ },
+ "payload": {
+ "metric": {
+ "type": "llm-judge",
+ "description": null,
+ "labels": {},
+ "supported_job_types": [
+ "online",
+ "offline"
+ ],
+ "model": {
+ "url": "https://integrate.api.nvidia.com/v1/chat/completions",
+ "name": "nvidia/nemotron-3-super-120b-a12b",
+ "host_url": null,
+ "api_key_secret": "NVIDIA_API_KEY",
+ "format": "nim"
+ },
+ "scores": [
{
+ "name": "helpfulness",
+ "description": "How well the response helps the user.",
+ "parser": {
+ "type": "json",
+ "json_path": "helpfulness"
+ },
+ "minimum": 0,
+ "maximum": 4
+ }
+ ],
+ "prompt_template": {
+ "messages": [
+ {
+ "role": "system",
+ "content": "Rate helpfulness from 0-4. Return JSON only: {\"helpfulness\": }."
+ },
+ {
"role": "user",
- "content": "{{item.input}}"
+ "content": "Request: {{item.input}}\nResponse: {{sample.output_text}}"
+ }
+ ]
+ },
+ "optional_fields": [],
+ "structured_output": {
+ "schema": {
+ "type": "object",
+ "properties": {
+ "helpfulness": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 4
+ }
+ },
+ "required": [
+ "helpfulness"
+ ]
}
- ]
+ },
+ "inference": null,
+ "system_prompt": null,
+ "reasoning": null,
+ "ignore_request_failure": false,
+ "job_type": "online"
+ },
+ "digest": "bc8712d3e93806036bb07b5f572fea60049149a6785a720c85303c17f04ea078",
+ "kind": "inline"
+ }
+ }
+ ],
+ "dataset": [
+ {
+ "input": "What is the capital of France?"
+ },
+ {
+ "input": "How do I make scrambled eggs?"
}
+ ],
+ "params": {
+ "parallelism": 2,
+ "limit_samples": 2,
+ "request_timeout": 120,
+ "max_retries": 3
+ },
+ "target": {
+ "url": "https://integrate.api.nvidia.com/v1/chat/completions",
+ "name": "nvidia/nemotron-3-super-120b-a12b",
+ "host_url": null,
+ "api_key_secret": "NVIDIA_API_KEY",
+ "format": "nim"
+ },
+ "prompt_template": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "{{item.input}}"
+ }
+ ]
+ }
}
diff --git a/skills/nemo-evaluator-plugin/references/agent-evaluation.md b/skills/nemo-evaluator-plugin/references/agent-evaluation.md
new file mode 100644
index 0000000000..d528ddb6c2
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/references/agent-evaluation.md
@@ -0,0 +1,226 @@
+# Agent Evaluation
+
+Read this file for agentic task-driven evaluation, direct SDK runners, platform
+`agent-evaluate` jobs, tasksets, precomputed trials, or Harbor and custom runners.
+
+## Choose standalone SDK or platform job
+
+Use `AgentEvaluator` for lightweight in-process evaluation that does not require a running nemo-platform:
+
+```python
+from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator
+
+result = await AgentEvaluator().run(tasks=tasks, target=target)
+print(result.trials)
+print(result.summary)
+```
+
+The standalone target union is:
+
+- `Model`
+- `GenericAgent`
+- Any object implementing `nemo_evaluator_sdk.agent_eval.trials.AgentTaskRunner` protocol
+
+For a minimal direct runner:
+
+```python
+from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import (
+ CallableAgentTaskRunner,
+)
+
+async def answer(task):
+ return task.inputs["instruction"]
+
+runner = CallableAgentTaskRunner(answer)
+result = await AgentEvaluator().run(tasks=tasks, target=runner)
+```
+
+Submit the plugin job when platform execution is required:
+
+```bash
+nemo evaluator agent-evaluate explain
+nemo evaluator agent-evaluate submit --spec-file agent-eval.json
+```
+
+## Build the job input
+
+`AgentEvalInputSpec.tasks` accepts an inline task list or a stored `TasksetRef`.
+Provide exactly one trial source:
+
+- `target` to generate trials.
+- `trials` to rescore precomputed trials.
+
+```python
+from nemo_evaluator.jobs.agent_spec import (
+ AgentEvalInputSpec,
+ AgentEvalTaskInput,
+ CodexRunnerTarget,
+)
+
+spec = AgentEvalInputSpec(
+ tasks=[
+ AgentEvalTaskInput(
+ id="capital-france",
+ intent="Name the capital of France.",
+ inputs={"instruction": "What is the capital of France?"},
+ metrics=[metric_bundle],
+ )
+ ],
+ target=CodexRunnerTarget(model=""),
+ max_concurrent_tasks=2,
+ fail_fast=False,
+ benchmark={"name": "geography-smoke"},
+)
+```
+
+Use `TasksetRef("default/geography")` with `submit` for persisted tasks. Stored
+tasks do not include grader-only `reference`; use inline tasks when metrics
+require held-out per-task data.
+
+## Choose a platform target
+
+| Target | Use when |
+| --- | --- |
+| `ModelTarget` | Generate trials through an OpenAI-compatible model endpoint |
+| `AgentTarget` | Generate trials through a generic HTTP or NeMo Agent Toolkit agent |
+| `CodexRunnerTarget` | Drive the Codex CLI runner |
+| `FabricRunnerTarget` | Run a configured NeMo [Fabric](https://github.com/nvidia/nemo-fabric) runner |
+| `HarborRunnerTarget` | Run a Harbor task suite in Docker |
+
+`ModelTarget` owns its `prompt_template` and online model params.
+`AgentTarget` owns its agent request configuration. Runner targets are resolved
+to an `AgentTaskRunner` inside the job runtime.
+
+For [Fabric](https://github.com/nvidia/nemo-fabric), pass one complete `agent.yaml` as a JSON-shaped `config`; the
+`harness.adapter_id` selects the harness:
+
+```python
+from nemo_evaluator.jobs.agent_spec import FabricRunnerTarget
+
+target = FabricRunnerTarget(
+ config={
+ "metadata": {"name": "regression-suite"},
+ "harness": {"adapter_id": "nvidia.fabric.codex"},
+ },
+ model="/",
+)
+```
+
+Do not use profile overlays. Fold the complete configuration into `config`.
+
+`max_concurrent_tasks` limits tasks evaluated concurrently. Target-specific
+settings such as inference parallelism or Harbor
+`n_concurrent_trials` control concurrency inside trial generation.
+
+## Use precomputed trials
+
+Pass `trials=[...]` and omit `target` to rescore stored outputs and/or trajectories
+without invoking the original model, agent, or runner. Keep stable `task_id`
+values so trials match task definitions.
+
+Individual trials are stored in the run bundle, not as queryable result entities.
+Retrieve the run index, download its bundle, and hydrate `trials.jsonl`:
+
+```python
+from nemo_evaluator_sdk.agent_eval.persistence import read_trials
+
+stored = client.evaluator.agent_eval_results.retrieve("")
+client.files.download(remote_path=stored.bundle_ref, local_path="previous-run")
+trials = read_trials("previous-run")
+```
+
+CLI equivalent for downloading the bundle:
+
+```bash
+nemo jobs results download agent-eval-results \
+ --job --output-file agent-eval-results.tar.gz
+mkdir -p previous-run
+tar -xzf agent-eval-results.tar.gz -C previous-run --strip-components=1
+```
+
+Pass the hydrated `trials` with the same task definitions and omit `target`.
+
+## Read results
+
+A standalone run returns an `AgentEvalResult`:
+
+- `result.summary` contains aggregate values per metric output plus coverage
+ counts for scored, failed, and missing-output trials.
+- `result.scores` contains one entry per task, trial, and metric, including
+ metric outputs, status, and diagnostics.
+- `result.trials` contains each agent output, its evidence, and its
+ `completed`, `partial`, or `failed` status.
+- `result.run_id` identifies the run; `result.benchmark` contains its grouping
+ metadata.
+
+When standalone `AgentEvalRunConfig.output_dir` is set, the same information is
+written as a run bundle:
+
+| File | Contents |
+| --- | --- |
+| `summary.json` | Aggregate mean, minimum, maximum, standard deviation, counts, and coverage |
+| `scores.jsonl` | Per-task, trial, and metric outputs, status, and diagnostics |
+| `trials.jsonl` | Trial outputs, evidence, metadata, and status |
+| `tasks.jsonl` | Tasks included in the run |
+| `run.json` | Run ID and artifact manifest |
+| `benchmark.json` | Benchmark-grouping metadata |
+| `report.html` | Browsable dashboard when dashboard generation is enabled |
+
+Use the in-memory result for programmatic follow-up and the bundle for
+inspection, sharing, or rescoring. Platform jobs persist the bundle and create
+a queryable record under `client.evaluator.agent_eval_results`.
+
+Inspect failed and partial trials and score diagnostics before interpreting
+aggregate values; a high mean with low coverage can hide missing or failed
+work.
+
+## Configure Harbor as a task runner
+
+Harbor requires its Python package, Docker access, and a Harbor dataset. Task
+discovery records the source dataset in each task's
+`harbor_dataset_path` metadata; the durable runtime recovers the dataset from
+that metadata.
+
+**Standalone SDK:**
+
+```python
+from pathlib import Path
+
+from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import (
+ HarborAgentTaskRunner,
+ HarborRuntimeConfig,
+ discover_harbor_tasks,
+)
+
+tasks = discover_harbor_tasks("path/to/harbor-suite")
+runner = HarborAgentTaskRunner(
+ config=HarborRuntimeConfig(
+ jobs_dir=Path("harbor-jobs"),
+ agent_name="oracle",
+ n_attempts=1,
+ n_concurrent_trials=2,
+ )
+)
+result = await AgentEvaluator().run(tasks=tasks, target=runner)
+```
+
+**Platform SDK:**
+
+```python
+from nemo_evaluator.jobs.agent_spec import HarborRunnerTarget
+
+target = HarborRunnerTarget(
+ agent_name="oracle",
+ n_attempts=1,
+ n_concurrent_trials=2,
+ max_retries=0,
+ artifacts=["/workspace/output"],
+ trace_dir="/app/traces",
+ reward_key="reward",
+)
+```
+
+Use `agent_import_path` for a custom Harbor agent and `agent_model_name` when
+the agent requires a model. The module must be importable in the execution
+environment. Durable execution additionally requires an execution image and
+runtime that provide Harbor and Docker access.
diff --git a/skills/nemo-evaluator-plugin/references/api-auth.md b/skills/nemo-evaluator-plugin/references/api-auth.md
index 4c69361724..e49701938b 100644
--- a/skills/nemo-evaluator-plugin/references/api-auth.md
+++ b/skills/nemo-evaluator-plugin/references/api-auth.md
@@ -1,15 +1,68 @@
# Evaluator API Auth
-Use the correct `model.api_key_secret` (if `model` is used) for the evaluator execution mode:
+Read this file before configuring a model, agent, remote metric, LLM judge, or
+durable platform job.
-- Local `nemo evaluator evaluate run`: `api_key_secret` is the name of an environment variable available to the local process, such as `NVIDIA_API_KEY`.
-- Remote `nemo evaluator evaluate submit`: `api_key_secret` is the name of a NeMo platform secret in the target workspace, such as `nvidia-api-key`.
+## Match the secret reference to the execution mode
-The remote job runtime cannot read local environment variables. In remote mode, if a model sets `api_key_secret`, create or verify the platform secret before submitting the job:
+`api_key_secret` is a reference, never the credential value. It resolves to a different value depending on execution mode: standalone and plugin submission.
+
+| Execution | `api_key_secret` resolves to |
+| --- | --- |
+| Standalone SDK | Environment-variable name in the calling process, such as `NVIDIA_API_KEY` |
+| Plugin `submit` | NeMo Platform secret name in the target workspace, such as `nvidia-api-key` |
+
+A remote job cannot read the submitting shell's environment variables. Before
+submitting, verify the `api_key_secret` is in the list of secrets:
+
+```bash
+nemo secrets list
+```
+
+Create it through the supported secrets CLI for the installed NeMo Platform
+version. Do not put the key directly in a spec, command line, log, or committed
+file.
```bash
printf '%s' "$NVIDIA_API_KEY" | nemo secrets create nvidia-api-key --from-file -
nemo secrets list
```
-If you copy a local LLM-judge spec that uses `"api_key_secret": "NVIDIA_API_KEY"` for remote submission, change that value to the platform secret name, for example `"nvidia-api-key"`.
+## Adapt the local-first spec for platform submission
+
+The checked `llm_as_judge.json` uses the local environment variable
+`NVIDIA_API_KEY`. Create a platform copy that points both the generation target
+and the judge's environment binding at the workspace secret:
+
+```bash
+jq --arg platform_secret "nvidia-api-key" '
+ .target.api_key_secret = $platform_secret
+ | .metrics[0].secrets.NVIDIA_API_KEY = $platform_secret
+' skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json \
+ > llm_as_judge.platform.json
+```
+
+The bundle key `NVIDIA_API_KEY` remains the environment-variable name expected
+by the judge; its value becomes the platform secret name. Do not edit
+`metrics[*].payload` or its digest will no longer describe the inline metric.
+
+## Diagnose remote 409 responses
+
+Do not assume HTTP 409 means a duplicate job. Inspect the response body. The
+Jobs service can return 409 when a referenced platform secret does not exist
+or is inaccessible, for example:
+
+```text
+Unable to create job because one or more referenced secrets were not found or
+are not accessible.
+```
+
+The response intentionally may not identify the secret. Verify every referenced
+workspace and secret name, then retry the submission.
+
+## Follow security best practices
+
+- Print secret names only, never values.
+- Redact authorization headers and provider responses that echo credentials.
+- Use placeholders such as `` in shared examples.
+- Do not copy `.env` files into job artifacts.
diff --git a/skills/nemo-evaluator-plugin/references/evaluation-shapes.md b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md
new file mode 100644
index 0000000000..6d04edc11b
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md
@@ -0,0 +1,42 @@
+# Dataset-Driven vs Task-Driven Evaluation
+
+Choose the evaluation shape from what produces the scored output. Metrics are
+shared scorers; the input and evidence differ.
+
+| Question | Dataset-driven | Task-driven |
+| --- | --- | --- |
+| What is the input? | Fixed dataset rows | Tasks with intent, inputs, and metrics |
+| What is scored? | One output per row | One or more trials per task |
+| Which metrics apply? | The same metric set applies to every row | Each task can define its own metrics |
+| What evidence is available? | Row fields, row scores, and aggregates | Final output, trajectory, tool calls, other trial evidence, per-task rewards, and summary |
+| Platform job | `evaluate submit` | `agent-evaluate submit` |
+
+## Dataset-driven evaluation
+
+Use dataset-driven evaluation for a fixed set of examples where the same
+scoring rules apply to every row. Typical uses include model quality checks,
+RAG regression tests, and labeled-set benchmarks.
+
+Each row contains the fields consumed by the metric, for example:
+
+```python
+{"question": "Capital of France?", "expected": "Paris", "output": "Paris"}
+```
+
+Read [SDK Execution](execution.md) for datasets, targets, field mappings,
+submission, and results.
+Read [Metric Selection](metric-selection.md) to choose
+the scorer.
+
+## Task-driven evaluation
+
+Use task-driven evaluation when the system performs work and the process can
+matter as much as the final answer. A model, agent, or runner produces a trial
+containing the final output and available execution evidence. Tasks can carry
+different metrics, so one taskset can grade heterogeneous work.
+
+Choose this shape for agent behavior, tool use, multi-step work, runner-based
+benchmarks, or rescoring precomputed trials.
+
+Read [Agent Evaluation](agent-evaluation.md) for tasks, trials, tasksets,
+targets, runners, concurrency, and results.
diff --git a/skills/nemo-evaluator-plugin/references/execution.md b/skills/nemo-evaluator-plugin/references/execution.md
new file mode 100644
index 0000000000..fdf5ac719f
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/references/execution.md
@@ -0,0 +1,268 @@
+# SDK Execution
+
+Read this file before choosing a dataset representation, execution mode,
+target, configuration, field mapping, or job-result operation.
+
+CLI snippets use the installed `nemo` command. In a repository checkout,
+prefix them with `uv run`.
+
+## Validate standalone, then submit to the platform
+
+### Standalone SDK
+
+Use the standalone SDK for the fastest in-process metric loop:
+
+```python
+from nemo_evaluator_sdk import Evaluator, ExactMatchMetric
+
+result = Evaluator().run_sync(
+ metrics=ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+ ),
+ dataset=[
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+ ],
+)
+print(result.row_scores)
+print(result.aggregate_scores)
+```
+
+**Platform CLI**
+
+Platform CLI equivalent for the same checked metric and rows:
+
+```bash
+uv run nemo evaluator evaluate submit \
+ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json
+```
+
+**Platform Python SDK**
+
+Use `client.evaluator.submit` for execution through the installed nemo-evaluator-plugin:
+
+```python
+from nemo_evaluator.sdk import RunConfig
+from nemo_evaluator_sdk import ExactMatchMetric
+from nemo_platform import NeMoPlatform
+
+client = NeMoPlatform(base_url="http://localhost:8080", workspace="default")
+job = client.evaluator.submit(
+ metric=ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+ ),
+ dataset=[
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+ ],
+ config=RunConfig(parallelism=2),
+)
+job.wait_until_done()
+result = job.get_result()
+```
+
+### Use a Fileset for the dataset
+
+The job will run in the container environment.
+The plugin submission dataset accepts inline rows, a `str` or `Path`, and
+`FilesetRef`:
+
+**Platform SDK**
+
+```python
+from nemo_evaluator.sdk import FilesetRef
+
+dataset = FilesetRef("default/eval-data")
+```
+
+**Platform CLI**
+
+CLI equivalent, using a stored metric and fileset:
+
+```bash
+nemo evaluator evaluate submit \
+ --spec '{"metrics":["default/exact-answer"],"dataset":"default/eval-data"}'
+```
+
+## Configure online generation
+
+Use `RunConfigOnlineModel` with `Model`, or `RunConfigOnline` with `Agent` when the evaluator should generate output before scoring.
+Provide a prompt template when the evaluator should generate output before
+scoring.
+
+**Platform SDK**
+
+```python
+from nemo_evaluator_sdk import ExactMatchMetric, Model, RunConfigOnlineModel
+
+job = client.evaluator.submit(
+ metric=ExactMatchMetric(reference="{{item.expected}}"),
+ dataset=[{"question": "Capital of France?", "expected": "Paris"}],
+ target=Model(
+ url="https://provider.example/v1/chat/completions",
+ name="",
+ format="openai",
+ api_key_secret="",
+ ),
+ prompt_template={
+ "messages": [{"role": "user", "content": "{{item.question}}"}],
+ },
+ config=RunConfigOnlineModel(parallelism=2),
+)
+job.wait_until_done()
+result = job.get_result()
+```
+
+**Platform CLI**
+
+The checked LLM-judge spec is local-first and reads `NVIDIA_API_KEY`. Follow
+[Adapt the local-first spec for platform submission](api-auth.md#adapt-the-local-first-spec-for-platform-submission)
+to create `llm_as_judge.platform.json`, then submit that copy:
+
+```bash
+nemo evaluator evaluate submit \
+ --spec-file llm_as_judge.platform.json
+```
+
+Platform submission requires provider access and the referenced platform workspace
+secret.
+
+## Map noncanonical fields
+
+Use `FieldMapping` when the metric expects canonical evaluator fields but the
+dataset uses different column names:
+
+**Platform SDK**
+
+```python
+from nemo_evaluator_sdk import FieldMapping
+
+mapping = FieldMapping(
+ output="assistant_answer",
+ reference="gold_answer",
+)
+```
+
+Pass it as `field_mapping=mapping` when submitting the job.
+
+**Platform CLI**
+
+```bash
+nemo evaluator evaluate submit --spec \
+ '{
+ "metrics": ["default/exact-answer"],
+ "dataset": [{"gold_answer": "Paris", "assistant_answer": "Paris"}],
+ "field_mapping": {
+ "output": "assistant_answer",
+ "reference": "gold_answer"
+ }
+ }'
+```
+
+Use field_mapping when a metric or online prompt uses canonical evaluator fields but the dataset uses different column names. One job-level mapping applies to every metric and the generation prompt.
+
+## Getting job results
+
+**Platform SDK**
+
+```python
+job = client.evaluator.submit(
+ metric=metric,
+ dataset=dataset,
+ config=config,
+ target=target,
+ prompt_template=prompt_template,
+)
+
+job.wait_until_done()
+result = job.get_result()
+artifacts = job.download_artifacts("evaluation-artifacts")
+```
+
+`EvaluatorJobResource` also exposes methods for job lifecycle management:
+
+- `name` and `job`
+- `get_job_status()`
+- `check_if_complete(raise_if_not_complete=False)`
+- `get_result(aggregate_fields=...)`
+- `as_async()`
+
+**Platform CLI**
+
+Poll until the job is completed before downloading results:
+
+```bash
+nemo evaluator evaluate submit --spec-file evaluation.json
+nemo jobs get-status
+nemo jobs results list
+nemo jobs results download aggregate-scores \
+ --job --output-file aggregate-scores.json
+nemo jobs results download row-scores \
+ --job --output-file row-scores.jsonl
+```
+
+The CLI `submit` command returns the created job record immediately. It does
+not wait or expose follow-up result/download commands under the `evaluate`
+group. Use the SDK submission handle when the workflow needs those lifecycle
+operations.
+
+### Notes
+
+- Always wait for terminal completion. A metric can report 100 percent progress
+before the platform finishes publishing result artifacts.
+- `submit` accepts a concrete `Model` or `ModelRef`; the platform resolves model
+references in the target workspace.
+
+## Multiple metrics
+
+The high-level plugin helper takes one runtime `metric`. To combine metrics,
+build an `EvaluateInputSpec` and submit it with the CLI. Stored metric references
+are resolved by the platform submission path:
+
+```json
+{
+ "metrics": [
+ "default/accuracy",
+ "default/style"
+ ],
+ "dataset": "default/eval-data",
+ "params": {"parallelism": 4}
+}
+```
+
+Save the spec as `multi-metric.json`, then submit it:
+
+```bash
+nemo evaluator evaluate submit --spec-file multi-metric.json
+```
+
+Inspect the authoritative wire schema before authoring a spec:
+
+```bash
+nemo evaluator evaluate explain
+```
+
+## Package metrics safely
+
+Built-in metrics default to declarative inline bundles. For a custom Python
+metric submitted to a service, opt in explicitly:
+
+```python
+from nemo_evaluator.shared.metric_bundles.hybrid import HybridMetricBundlePackager
+
+job = client.evaluator.submit(
+ metric=custom_metric,
+ dataset=rows,
+ metric_bundle_packager=HybridMetricBundlePackager(),
+)
+```
+
+The CLI cannot package a Python metric object or select
+`metric_bundle_packager`. After Python serializes the bundled metric into a
+complete spec, submit that spec with:
+
+```bash
+nemo evaluator evaluate submit --spec-file custom-metric.json
+```
diff --git a/skills/nemo-evaluator-plugin/references/llm-judge.md b/skills/nemo-evaluator-plugin/references/llm-judge.md
index 2052924ca1..b5d0ce6fe3 100644
--- a/skills/nemo-evaluator-plugin/references/llm-judge.md
+++ b/skills/nemo-evaluator-plugin/references/llm-judge.md
@@ -1,32 +1,78 @@
-# LLM Judge Notes
+# LLM Judge
-Use `nemo evaluator evaluate explain` to inspect the current Evaluator plugin spec schema before creating an LLM-judge run.
+Read this file when deterministic metrics cannot express the rubric and an LLM
+must score existing or generated responses.
-When configuring an LLM judge, verify:
+## Configure the judge
-1. The judge model authentication reference matches the execution mode. See [Evaluator API Auth](api-auth.md).
+Keep the judge model, score contract, parser, and prompt explicit:
-2. The judge model name is the API model ID expected by the endpoint, not an entity display name.
+```python
+from nemo_evaluator_sdk import JSONScoreParser, LLMJudgeMetric, Model, RangeScore
-3. The metric prompt and parser match the output you expect from the judge model.
+judge = LLMJudgeMetric(
+ model=Model(
+ url="https://provider.example/v1/chat/completions",
+ name="",
+ format="openai",
+ api_key_secret="",
+ ),
+ scores=[
+ RangeScore(
+ name="helpfulness",
+ description="How well the response addresses the request.",
+ minimum=0,
+ maximum=4,
+ parser=JSONScoreParser(json_path="helpfulness"),
+ )
+ ],
+ prompt_template={
+ "messages": [
+ {
+ "role": "system",
+ "content": 'Return JSON only: {"helpfulness": }.',
+ },
+ {
+ "role": "user",
+ "content": "Request: {{item.input}}\nResponse: {{item.output}}",
+ },
+ ]
+ },
+)
+```
-For local iteration, keep the metric and dataset in a spec file and run:
+Use lowercase letters, numbers, and underscores in score names. Ensure the
+judge response exactly matches the parser: the example parser expects a JSON
+field named `helpfulness`.
-```bash
-nemo evaluator evaluate run --spec-file evaluation-spec.json
-```
+## Validate before scaling
-The checked-in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json` is a local-run example. It expects `NVIDIA_API_KEY` to be set in the local shell.
+1. Use one response that should score high and one that should score low.
+2. Confirm the model ID is accepted by the configured endpoint.
+3. Inspect raw judge output and row-level parser errors.
+4. Confirm the score range and aggregate match the rubric.
+5. Only then increase dataset size or submit a durable job.
-For durable execution, submit the same spec:
+Use:
```bash
-nemo evaluator evaluate submit \
- --spec-file evaluation-spec.json \
- --workspace default \
- --profile default
+nemo evaluator metric-types llm-judge
+nemo evaluator evaluate explain
```
-Before submitting an LLM-judge spec via `submit`, replace local environment-variable names with platform secret names, such as `nvidia-api-key`.
+Prefer `--spec-file` over shell-escaped inline JSON. The checked
+`assets/specs/llm_as_judge.json` demonstrates the minimum online generation
+target plus judge configuration.
+
+## Keep judge and generation roles separate
+
+For offline judge-quality evaluation, put existing responses in dataset rows
+and omit the generation target. For online generation-quality evaluation, pass
+a separate `Model` or `Agent` target plus `prompt_template`; the judge metric
+then scores the generated sample.
+
+Do not treat labels for old responses as labels for newly generated responses
+unless the benchmark protocol explicitly defines that mapping.
-Prefer `--spec-file` over inline `--spec` for LLM-judge metrics because prompts and score definitions quickly become hard to audit as shell-escaped JSON.
+See [Evaluator API Auth](api-auth.md) before switching the same configuration
+between standalone and platform execution.
diff --git a/skills/nemo-evaluator-plugin/references/metric-selection.md b/skills/nemo-evaluator-plugin/references/metric-selection.md
new file mode 100644
index 0000000000..4f5647460c
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/references/metric-selection.md
@@ -0,0 +1,82 @@
+# Metric Selection
+
+Read this file when converting a rubric into evaluator metrics.
+
+## Prefer the simplest metric
+Python import path of a metric: `nemo_evaluator_sdk.metrics.`
+
+| Goal | Prefer |
+| --- | --- |
+| Exact label, enum, or regression | `ExactMatchMetric` |
+| Contains, equals, or starts/ends with | `StringCheckMetric` |
+| Numeric value or threshold | `NumberCheckMetric` |
+| Text overlap | `F1Metric`, `BLEUMetric`, or `ROUGEMetric` |
+| Semantic quality or a written rubric | `LLMJudgeMetric` |
+| Retrieval smoke test | A deterministic context assertion or `LLMJudgeMetric` |
+| Tool-call correctness | `ToolCallingMetric` or `ToolCallAccuracyMetric` |
+| Existing scoring service | `RemoteMetric` or `NemoAgentToolkitRemoteMetric` |
+| Agent answer or goal completion | A task-specific custom metric that implements `nemo_evaluator_sdk.metrics.protocol.Metric` or `LLMJudgeMetric` |
+
+Use deterministic metrics before an LLM judge. Use an LLM only when the
+criterion requires semantic judgment.
+
+RAGAS metrics remain available for specialized RAG evaluation, but do not use
+them as skill smoke examples: even a one-row live judge call can take minutes.
+Use the evaluator RAG metric documentation when the user explicitly requests a
+RAGAS score, validate its input schema first, and label the provider and
+timeout prerequisites.
+
+### Explore the metrics provided by the SDK
+
+List current metric names and inspect one schema:
+
+```bash
+uv run nemo evaluator metric-types
+uv run nemo evaluator metric-types exact-match
+```
+
+## Validate the mapping
+
+Create one row that must pass and one that must fail:
+
+```python
+from nemo_evaluator_sdk import ExactMatchMetric
+
+metric = ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+)
+rows = [
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+]
+```
+
+Check:
+
+- Dataset keys match every Jinja template.
+- Normalization is intentional; do not hide case or whitespace differences
+ unless the rubric says they are irrelevant.
+- Judge prompts specify the rubric and parser-compatible output.
+- RAG and tool metrics receive their required canonical fields or a
+ `FieldMapping`.
+
+## Use multiple metrics only for distinct dimensions
+
+SDK — pass a metric sequence in one call:
+
+```python
+from nemo_evaluator_sdk import Evaluator
+result = Evaluator().run_sync(metrics=[accuracy, style], dataset=rows)
+```
+
+Platform job — put multiple stored metrics on the job spec:
+
+```bash
+uv run nemo evaluator evaluate submit --spec \
+ '{"metrics":["default/accuracy","default/style"],"dataset":"default/eval-data"}'
+```
+
+Each `metrics` entry may be an inline metric bundle, a stored `MetricRef`, or
+a mix of both. The high-level `client.evaluator.submit` helper still accepts
+only one runtime metric per call.
diff --git a/skills/nemo-evaluator-plugin/references/resources.md b/skills/nemo-evaluator-plugin/references/resources.md
new file mode 100644
index 0000000000..826b02794e
--- /dev/null
+++ b/skills/nemo-evaluator-plugin/references/resources.md
@@ -0,0 +1,135 @@
+# Stored Resources
+
+Read this file when definitions or results must be reusable and queryable
+through `client.evaluator`.
+
+## Resource map
+
+| Resource | Create | Retrieve | List | Delete | Update |
+| --- | --- | --- | --- | --- | --- |
+| `metrics` | yes | yes | yes | yes | no |
+| `tasks` | yes | yes | yes | yes | no |
+| `tasksets` | yes | yes | yes | yes | no |
+| `eval_results` | no | yes | yes | yes | no |
+| `agent_eval_results` | no | yes | yes | yes | no |
+
+Metrics, tasks, and tasksets are immutable. Delete and recreate them, or use a
+new versioned name.
+
+## Store a metric, task, and taskset
+
+```python
+from nemo_evaluator.api.schemas import (
+ MetricRef,
+ TaskInput,
+ TaskInputs,
+ TaskRef,
+ TasksetInput,
+)
+from nemo_evaluator_sdk import StringCheckMetric
+from nemo_platform import NeMoPlatform
+
+client = NeMoPlatform(base_url="", workspace="")
+
+client.evaluator.metrics.create(
+ "answer-exact",
+ metric=StringCheckMetric(
+ operation="equals",
+ left_template="{{sample.output_text | trim}}",
+ right_template="Paris",
+ ),
+)
+
+client.evaluator.tasks.create(
+ "capital-france",
+ task=TaskInput(
+ intent="Name the capital of France.",
+ inputs=TaskInputs(instruction="What is the capital of France?"),
+ metrics=[MetricRef("default/answer-exact")],
+ ),
+)
+
+client.evaluator.tasksets.create(
+ "geography",
+ taskset=TasksetInput(
+ description="Geography smoke tasks.",
+ tasks=[TaskRef("default/capital-france")],
+ ),
+)
+```
+
+For a task that needs held-out ground truth invisible to the agent, keep the reference on an
+inline `AgentEvalTaskInput` and use a metric that reads it:
+
+```python
+from nemo_evaluator.api.schemas import MetricRef, TaskInputs
+from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput
+from nemo_evaluator_sdk import ExactMatchMetric
+
+client.evaluator.metrics.create(
+ "answer-from-reference",
+ metric=ExactMatchMetric(
+ reference="{{reference.expected}}",
+ candidate="{{sample.output_text}}",
+ ),
+)
+
+inline_task = AgentEvalTaskInput(
+ id="capital-france",
+ intent="Name the capital of France.",
+ inputs=TaskInputs(instruction="What is the capital of France?"),
+ reference={"expected": "Paris"},
+ metrics=[MetricRef("default/answer-from-reference")],
+)
+```
+
+Stored tasks keep metric references. Inline task metrics are normalized into
+content-addressed derived metrics. The stored-task example uses an output-only
+metric because stored tasks do not carry the grader-only `reference` field; use
+an inline `AgentEvalTaskInput` when held-out per-task data is required.
+
+## Retrieve, list, and delete
+
+```python
+metric = client.evaluator.metrics.retrieve("answer-exact")
+metrics = client.evaluator.metrics.list(metric_type="string-check")
+tasks = client.evaluator.tasks.list(page=1, page_size=100, sort="name")
+tasksets = client.evaluator.tasksets.list(page=1, page_size=100)
+
+client.evaluator.tasksets.delete("geography")
+client.evaluator.tasks.delete("capital-france")
+client.evaluator.metrics.delete("answer-exact")
+```
+
+Metric listing supports `metric_type` and `include_derived`. Task and taskset
+listing support pagination and sorting. Every method accepts an optional
+`workspace`; create operations also accept `project`.
+
+## Query persisted results
+
+Dataset-driven durable jobs create `eval_results`; agent-evaluation jobs create
+`agent_eval_results`.
+
+```python
+row_eval = client.evaluator.eval_results.retrieve("")
+row_page = client.evaluator.eval_results.list(
+ job_id="",
+ target_kind="model",
+ target_name="",
+ dataset_ref="default/eval-data",
+)
+
+agent_eval = client.evaluator.agent_eval_results.retrieve("")
+agent_page = client.evaluator.agent_eval_results.list(
+ job_id="",
+ target_kind="harbor",
+ target_name="oracle",
+)
+```
+
+Both result resources support pagination, sorting, workspace override, and
+delete. Result indexing is best effort and separate from the authoritative job
+artifacts. Retry a short-lived `404`; if the record remains absent, inspect the
+job logs and use the artifact bundle. A persisted record is a queryable
+summary/index; use the bundle for complete row scores, trials, evidence, and
+reports.
diff --git a/skills/nemo-evaluator-plugin/references/troubleshooting.md b/skills/nemo-evaluator-plugin/references/troubleshooting.md
index e00f8609d0..ed7348e189 100644
--- a/skills/nemo-evaluator-plugin/references/troubleshooting.md
+++ b/skills/nemo-evaluator-plugin/references/troubleshooting.md
@@ -1,38 +1,40 @@
# Evaluation Troubleshooting
-The Evaluator plugin CLI surface is `nemo evaluator`.
+The plugin CLI surface is `nemo evaluator`. In a repository checkout, prefix
+the commands below with `uv run`.
-## Quick Checks
+## Inspect the installed contracts
```bash
-nemo evaluator --help
-nemo evaluator evaluate --help
+nemo evaluator info
+nemo evaluator metric-types
nemo evaluator evaluate explain
+nemo evaluator agent-evaluate explain
```
-## Local vs Cluster Runs
-
-Use local execution to validate the spec:
-
-```bash
-nemo evaluator evaluate run --spec-file evaluation-spec.json
-```
-
-Use cluster submission once the same spec works locally:
-
-```bash
-nemo evaluator evaluate submit \
- --spec-file evaluation-spec.json \
- --workspace default \
- --profile default
-```
-
-## Common Issues
-
-| Symptom | Cause | Fix |
-|---------|-------|-----|
-| `No such command 'evaluation'` | The legacy generated CLI group was removed | Use `nemo evaluator ...` |
-| Spec validation error | The submitted spec does not match the plugin schema | Run `nemo evaluator evaluate explain` and update the spec |
-| Secret not found during `submit` | The judge metric references a missing NeMo platform secret | Run `nemo secrets list` in the target workspace and create the secret if needed |
-| Local `run` cannot authenticate to the judge endpoint | `api_key_secret` points at a NeMo secret name instead of a local environment variable, or the environment variable is unset | Set the API key in the local environment and use that variable name as `api_key_secret`. See [Evaluator API Auth](api-auth.md) |
-| Local run works but submit fails | Cluster/profile/workspace configuration issue | Check `nemo evaluator evaluate submit --help`, then retry with explicit `--workspace`, `--profile`, and cluster options |
+## Common failures
+
+| Symptom | Likely cause | Fix |
+| --- | --- | --- |
+| `No such command 'evaluation'` | The legacy generated CLI group is not the plugin surface | Use `nemo evaluator ...` |
+| Spec validation error | Fields do not match the current job schema | Run the matching `explain` command and validate against the spec class before submission |
+| Dataset row has missing fields | Jinja templates or `field_mapping` do not match row keys | Inspect one row and every referenced template before rerunning |
+| Standalone model/agent authentication fails | `api_key_secret` names a platform secret instead of an environment variable, or the variable is unset | Use an environment-variable name; see [API Auth](api-auth.md) |
+| Remote submission returns 409 | The response may describe a missing platform secret, not a duplicate job | Read the response body and verify the workspace secret |
+| Built-in metric bundle contains cloudpickle | A legacy or explicit packager was used | Regenerate with `InlineMetricBundlePackager` or the current default |
+| `cloudpickle metric payload was created with Python ...` (HTTP 422) | The bundle was created with a different Python major/minor runtime | For a built-in metric, regenerate the checked inline JSON spec; for an intentional custom metric, recreate the bundle with the worker's Python major/minor version |
+| Custom metric submission rejects the default packager | Shipping custom code requires explicit opt-in | Pass `HybridMetricBundlePackager()` (preferred) or `CloudpickleMetricBundlePackager()` |
+| `ModelRef` fails with the standalone SDK | Model references are resolved by the platform submission path | Use a concrete `Model` with the standalone SDK or use `submit` with `ModelRef` |
+| Fileset evaluation cannot load data | The reference, fragment, or workspace is wrong | Verify the `FilesetRef` and access it through the same workspace |
+| Result download fails while progress shows 100% | Metric progress finished before the platform job finalized artifacts | Call `job.wait_until_done()` before `get_result()` or `download_artifacts()` |
+| Agent-eval rejects the spec | Both or neither of `target` and `trials` were provided | Provide exactly one |
+| Taskset evaluation lacks held-out reference data | Stored tasks do not carry grader-only `reference` | Use inline `AgentEvalTaskInput` when the metric needs held-out per-task data |
+| Runner target fails to start | The runtime dependency, CLI, config, credentials, or Docker access is missing | Check the selected Codex, Fabric, or Harbor runner prerequisites |
+
+## Debug in the smallest scope
+
+1. Validate one expected pass and one expected failure.
+2. Inspect row scores or task trials before aggregates.
+3. Reproduce metric behavior with the standalone SDK before diagnosing platform infrastructure.
+4. For submitted jobs, inspect terminal status and error details.
+5. Retry only the failed row, task, or runner configuration when possible.
diff --git a/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py
index 953454e8e3..623ce22223 100644
--- a/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py
+++ b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py
@@ -2,55 +2,220 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Print an exact-match metric bundle example.
+"""Generate or check the Evaluator plugin skill's checked JSON specs.
-Run from the repo root:
+Run from the repository root:
- uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py
+ uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py --check
+ uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py --write
"""
from __future__ import annotations
+import argparse
import json
-import os
-import sys
+from collections.abc import Callable
+from pathlib import Path
from typing import Any
-DETERMINISTIC_HASH_SEED = "0"
-JSON_OUTPUT_INDENT = 4
-SUCCESS_EXIT_CODE = 0
-
-
-def _ensure_deterministic_hash_seed() -> None:
- if os.environ.get("PYTHONHASHSEED") == DETERMINISTIC_HASH_SEED:
- return
- env = {**os.environ, "PYTHONHASHSEED": DETERMINISTIC_HASH_SEED}
- os.execvpe(sys.executable, [sys.executable, *sys.argv], env)
+from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec
+from nemo_evaluator.jobs.evaluate import EvaluateInputSpec
+from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric
+from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager
+from nemo_evaluator_sdk import (
+ ExactMatchMetric,
+ JSONScoreParser,
+ LLMJudgeMetric,
+ Model,
+ RangeScore,
+ SecretRef,
+)
+from nemo_evaluator_sdk.enums import ModelFormat
+
+SKILL_DIR = Path(__file__).resolve().parents[1]
+SPEC_DIR = SKILL_DIR / "assets" / "specs"
+SpecBuilder = Callable[[], dict[str, Any]]
def _bundle(metric: Any) -> dict[str, Any]:
- _ensure_deterministic_hash_seed()
-
- from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric
- from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager
-
- return bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json")
-
-
-def build_metric_bundle_example() -> dict[str, Any]:
- """Return bundled JSON for one configured SDK metric."""
- from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric
-
- metric = ExactMatchMetric(
- reference="{{item.gold_answer}}",
- candidate="{{item.prediction}}",
+ return bundle_metric(metric, InlineMetricBundlePackager()).model_dump(mode="json")
+
+
+def build_exact_match_spec() -> dict[str, Any]:
+ """Return a two-row offline exact-match spec."""
+ return {
+ "metrics": [
+ _bundle(
+ ExactMatchMetric(
+ reference="{{item.expected}}",
+ candidate="{{item.output}}",
+ )
+ )
+ ],
+ "dataset": [
+ {"expected": "Paris", "output": "Paris"},
+ {"expected": "Paris", "output": "London"},
+ ],
+ "params": {"parallelism": 2},
+ }
+
+
+def build_llm_as_judge_spec() -> dict[str, Any]:
+ """Return a minimal online generation plus LLM-judge spec."""
+ model = Model(
+ url="https://integrate.api.nvidia.com/v1/chat/completions",
+ name="nvidia/nemotron-3-super-120b-a12b",
+ api_key_secret=SecretRef(root="NVIDIA_API_KEY"),
+ format=ModelFormat.NVIDIA_NIM,
)
- return _bundle(metric)
-
-
-def main() -> int:
- print(json.dumps(build_metric_bundle_example(), indent=JSON_OUTPUT_INDENT))
- return SUCCESS_EXIT_CODE
+ judge = LLMJudgeMetric(
+ model=model,
+ scores=[
+ RangeScore(
+ name="helpfulness",
+ description="How well the response helps the user.",
+ minimum=0,
+ maximum=4,
+ parser=JSONScoreParser(json_path="helpfulness"),
+ )
+ ],
+ prompt_template={
+ "messages": [
+ {
+ "role": "system",
+ "content": 'Rate helpfulness from 0-4. Return JSON only: {"helpfulness": }.',
+ },
+ {
+ "role": "user",
+ "content": "Request: {{item.input}}\nResponse: {{sample.output_text}}",
+ },
+ ]
+ },
+ )
+ return {
+ "metrics": [_bundle(judge)],
+ "dataset": [
+ {"input": "What is the capital of France?"},
+ {"input": "How do I make scrambled eggs?"},
+ ],
+ "params": {
+ "parallelism": 2,
+ "limit_samples": 2,
+ "request_timeout": 120,
+ "max_retries": 3,
+ },
+ "target": model.model_dump(mode="json"),
+ "prompt_template": {
+ "messages": [{"role": "user", "content": "{{item.input}}"}],
+ },
+ }
+
+
+def build_fabric_agent_eval_spec() -> dict[str, Any]:
+ """Return a one-task durable Fabric agent-evaluation spec."""
+ return {
+ "tasks": [
+ {
+ "id": "capital-france",
+ "intent": "Name the capital of France.",
+ "inputs": {"instruction": "What is the capital of France?"},
+ "reference": {"expected": "Paris"},
+ "metrics": [
+ _bundle(
+ ExactMatchMetric(
+ reference="{{reference.expected}}",
+ candidate="{{sample.output_text}}",
+ )
+ )
+ ],
+ }
+ ],
+ "target": {
+ "kind": "fabric",
+ "config": {
+ "metadata": {"name": "readme-fabric-smoke"},
+ "harness": {"adapter_id": "nvidia.fabric.codex"},
+ },
+ "model": "/",
+ "capture_trajectory": False,
+ },
+ "max_concurrent_tasks": 1,
+ "fail_fast": True,
+ "benchmark": {"name": "readme-fabric-smoke"},
+ }
+
+
+SPEC_BUILDERS: dict[str, SpecBuilder] = {
+ "exact_match_metric.json": build_exact_match_spec,
+ "llm_as_judge.json": build_llm_as_judge_spec,
+}
+
+AGENT_SPEC_BUILDERS: dict[str, SpecBuilder] = {
+ "fabric_agent_eval.json": build_fabric_agent_eval_spec,
+}
+
+
+def generated_specs() -> dict[Path, dict[str, Any]]:
+ """Build and validate every checked dataset-evaluation spec."""
+ specs: dict[Path, dict[str, Any]] = {}
+ for name, builder in SPEC_BUILDERS.items():
+ payload = builder()
+ EvaluateInputSpec.model_validate(payload)
+ specs[SPEC_DIR / name] = payload
+ return specs
+
+
+def generated_agent_specs() -> dict[Path, dict[str, Any]]:
+ """Build and validate every checked agent-evaluation spec."""
+ specs: dict[Path, dict[str, Any]] = {}
+ for name, builder in AGENT_SPEC_BUILDERS.items():
+ payload = builder()
+ AgentEvalInputSpec.model_validate(payload)
+ specs[SPEC_DIR / name] = payload
+ return specs
+
+
+def _all_generated_specs() -> dict[Path, dict[str, Any]]:
+ return {**generated_specs(), **generated_agent_specs()}
+
+
+def _render(payload: dict[str, Any]) -> str:
+ return json.dumps(payload, indent=2) + "\n"
+
+
+def write_specs() -> int:
+ """Write generated specs to their checked locations."""
+ for path, payload in _all_generated_specs().items():
+ path.write_text(_render(payload), encoding="utf-8")
+ print(f"wrote {path.relative_to(SKILL_DIR)}")
+ return 0
+
+
+def check_specs() -> int:
+ """Return nonzero when a checked spec differs from generated output."""
+ stale: list[Path] = []
+ for path, payload in _all_generated_specs().items():
+ if not path.is_file() or path.read_text(encoding="utf-8") != _render(payload):
+ stale.append(path)
+
+ if stale:
+ for path in stale:
+ print(f"out of date: {path.relative_to(SKILL_DIR)}")
+ print("run with --write to refresh the checked specs")
+ return 1
+
+ count = len(SPEC_BUILDERS) + len(AGENT_SPEC_BUILDERS)
+ print(f"{count} evaluator example specs are up to date")
+ return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ action = parser.add_mutually_exclusive_group(required=True)
+ action.add_argument("--check", action="store_true", help="Check generated specs without writing.")
+ action.add_argument("--write", action="store_true", help="Write generated specs.")
+ args = parser.parse_args(argv)
+ return check_specs() if args.check else write_specs()
if __name__ == "__main__":