From 7088b202fb7dfebfd5587154a49a6e331a177438 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Mon, 3 Aug 2026 16:33:56 -0700 Subject: [PATCH 1/4] refactor: update the skills for eval plugin Signed-off-by: Nick Goncharenko --- docs/evaluator/agent-eval/index.mdx | 17 +- docs/evaluator/index.mdx | 10 +- .../examples/fabric_harness_runtimes.py | 13 +- .../agent_eval/test_fabric_integration.py | 41 ++- plugins/nemo-evaluator/README.md | 196 +++++++---- .../nemo-evaluator/src/nemo_evaluator/cli.py | 2 +- .../src/nemo_evaluator/docs/index.md | 179 ++++++---- .../nemo-evaluator/tests/test_evaluate_job.py | 170 +--------- .../tests/test_skill_examples.py | 316 ++++++++++++++++++ script/dev-install-fabric.sh | 20 +- skills/nemo-evaluator-plugin/SKILL.md | 197 +++++------ .../assets/examples/plugin_sdk_examples.py | 166 +++++---- .../assets/specs/exact_match_benchmark.json | 96 ------ .../assets/specs/exact_match_metric.json | 87 ++--- .../assets/specs/fabric_agent_eval.json | 70 ++++ .../assets/specs/llm_as_judge.json | 168 +++++++--- .../references/agent-evaluation.md | 226 +++++++++++++ .../references/api-auth.md | 63 +++- .../references/evaluation-shapes.md | 42 +++ .../references/execution.md | 253 ++++++++++++++ .../references/llm-judge.md | 82 ++++- .../references/metric-selection.md | 82 +++++ .../references/resources.md | 132 ++++++++ .../references/troubleshooting.md | 61 ++-- .../scripts/generate_example_specs.py | 237 +++++++++++-- 25 files changed, 2123 insertions(+), 803 deletions(-) create mode 100644 plugins/nemo-evaluator/tests/test_skill_examples.py delete mode 100644 skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json create mode 100644 skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json create mode 100644 skills/nemo-evaluator-plugin/references/agent-evaluation.md create mode 100644 skills/nemo-evaluator-plugin/references/evaluation-shapes.md create mode 100644 skills/nemo-evaluator-plugin/references/execution.md create mode 100644 skills/nemo-evaluator-plugin/references/metric-selection.md create mode 100644 skills/nemo-evaluator-plugin/references/resources.md diff --git a/docs/evaluator/agent-eval/index.mdx b/docs/evaluator/agent-eval/index.mdx index 377f7ceb9d..ccd50d8f53 100644 --- a/docs/evaluator/agent-eval/index.mdx +++ b/docs/evaluator/agent-eval/index.mdx @@ -13,10 +13,12 @@ got there*. Each task carries its own metrics, so a single suite can grade heter -**Platform-plugin support is in progress.** The pages in this section run agent evaluations through -the **local SDK** (`AgentEvaluator().run()`). Running them through the **NeMo Platform plugin** as -durable platform jobs — the way [dataset-driven metrics](/documentation/evaluate-models/metrics) -already can — is under active development; for now, use the local SDK path shown here. +**Local and platform durable interfaces are available.** +- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +that do not require running nemo-platform. +- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline +tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces +above remain dataset-driven only. @@ -84,9 +86,10 @@ print(result.summary) - the **trajectory** — how the agent worked (its tool use and steps); - **views** — named roll-ups you define on a task that combine two or more of its metric outputs into one reported score (for example, averaging an accuracy metric and a tool-use metric into a single `quality` score); - the **run-level aggregate** — results also roll up across the whole run. -- **Runs locally.** A full run — including the `report.html` dashboard — is produced on your machine - with no platform services required. (Running the same suite as a durable platform job through the - evaluator plugin is in progress — see the note above.) +- **Runs locally or durably.** A local run can produce the full bundle, + including `report.html`, without platform services. Use the plugin's + `agent-evaluate` job when the task suite needs durable platform execution and + persisted result metadata. - **Measurement, not decisions.** The evaluator produces scores, aggregates, and provenance — it doesn't decide pass/fail, gate a release, or compare runs. Those decisions belong to whatever consumes the results. diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index ab124840ad..7d247118ff 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -120,10 +120,12 @@ result = job.get_result() -**Agent evaluation runs from the SDK today.** Task-driven runs use the local SDK -(`AgentEvaluator().run()`); running them as durable platform jobs — the way dataset-driven metrics -already can — is [in progress](/documentation/evaluate-models/agent-eval). Use the local SDK path for -now. +**Agent evaluation has local and platform durable interfaces.** +- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +that do not require running nemo-platform. +- Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline +tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces +above remain dataset-driven only. diff --git a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py index e2e47385d9..a499f4fe61 100644 --- a/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py +++ b/packages/nemo_evaluator_sdk/examples/fabric_harness_runtimes.py @@ -9,7 +9,7 @@ ``adapter_id``, ``runtime.transport``, and any harness-specific ``harness.settings``: * **Codex CLI** (``nvidia.fabric.codex``) runs the agent as a subprocess — ``transport="cli"`` — - and takes codex-specific ``harness.settings`` (sandbox mode, git-repo check, ...). + and takes codex-specific ``harness.settings`` such as sandbox and approval modes. * **Hermes SDK** (``nvidia.fabric.hermes``) runs in-library — ``transport="library"`` — and declares its ``input``/``output`` schemas instead. @@ -28,10 +28,11 @@ import json from nemo_evaluator_sdk.agent_eval.runtimes.fabric.runtime import FabricAgentRuntime -from nemo_fabric import ( # ty: ignore[unresolved-import] +from nemo_fabric import ( FabricConfig, HarnessConfig, MetadataConfig, + ModelConfig, RuntimeConfig, ) @@ -40,9 +41,9 @@ metadata=MetadataConfig(name="codex-eval"), harness=HarnessConfig( adapter_id="nvidia.fabric.codex", - settings={"sandbox": "read-only", "skip_git_repo_check": True}, + settings={"sandbox": "read-only"}, ), - models={"default": {"provider": "openai", "model": "gpt-5.4"}}, + models={"default": ModelConfig(provider="openai", model="gpt-5.4")}, runtime=RuntimeConfig.from_mapping({"mode": "oneshot", "transport": "cli"}), ) @@ -50,7 +51,7 @@ HERMES_SDK_CONFIG = FabricConfig( metadata=MetadataConfig(name="hermes-eval"), harness=HarnessConfig(adapter_id="nvidia.fabric.hermes", resolution="preinstalled"), - models={"default": {"provider": "nvidia", "model": "qwen2.5-coder-32b"}}, + models={"default": ModelConfig(provider="nvidia", model="qwen2.5-coder-32b")}, runtime=RuntimeConfig.from_mapping( {"mode": "oneshot", "transport": "library", "input_schema": "chat", "output_schema": "message"} ), @@ -65,7 +66,7 @@ def build_runtime(harness: str, *, model: str | None = None, work_root: str | None = None) -> FabricAgentRuntime: """Build a :class:`FabricAgentRuntime` for a named harness (see :data:`HARNESS_CONFIGS`).""" - return FabricAgentRuntime(config=HARNESS_CONFIGS[harness], model=model, work_root=work_root) + return FabricAgentRuntime(config=HARNESS_CONFIGS[harness].to_mapping(), model=model, work_root=work_root) def main() -> None: diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py index 9c12c223af..68f7da3e40 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py @@ -8,8 +8,9 @@ in CI: it proves the runner -> evaluator -> metric -> evidence chain, i.e. the metric receives and reads the trajectory (ATIF) evidence for the task. - ``test_fabric_codex_live_eval_captures_atif_trajectory`` is the real fabric->codex->Relay run, gated - behind the required binaries so CI skips it; run it locally after ``uv sync --extra fabric`` - plus ``script/dev-install-fabric.sh`` for the relay gateway. + behind the required binaries so CI skips it; run it locally after + ``uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact`` plus + ``script/dev-install-fabric.sh`` for the relay gateway. """ from __future__ import annotations @@ -187,16 +188,16 @@ def __init__(self, **kwargs: Any) -> None: self.__dict__.update(kwargs) module = types.ModuleType("nemo_fabric") - module.Fabric = _FakeClient # type: ignore[attr-defined] - module.FabricConfig = _FakeConfig # type: ignore[attr-defined] - module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined] - module.ModelConfig = _FakeModelConfig # type: ignore[attr-defined] - module.RunRequest = _FakeRunRequest # type: ignore[attr-defined] + setattr(module, "Fabric", _FakeClient) + setattr(module, "FabricConfig", _FakeConfig) + setattr(module, "EnvironmentConfig", _FakeEnvironment) + setattr(module, "ModelConfig", _FakeModelConfig) + setattr(module, "RunRequest", _FakeRunRequest) # The runtime builds the relay observability config from Fabric's own typed models (lazy import). - module.RelayObservabilityConfig = _FakeRelayModel # type: ignore[attr-defined] - module.RelayAtifConfig = _FakeRelayModel # type: ignore[attr-defined] - module.RelayAtofConfig = _FakeRelayModel # type: ignore[attr-defined] - module.RelayAtofFileSinkConfig = _FakeRelayModel # type: ignore[attr-defined] + setattr(module, "RelayObservabilityConfig", _FakeRelayModel) + setattr(module, "RelayAtifConfig", _FakeRelayModel) + setattr(module, "RelayAtofConfig", _FakeRelayModel) + setattr(module, "RelayAtofFileSinkConfig", _FakeRelayModel) monkeypatch.setitem(sys.modules, "nemo_fabric", module) # nemo_relay stays a hard (installed) dependency here so ``run_tasks``'s capture-trajectory fail-fast # (``import nemo_relay.observability``) resolves; only the optional native nemo_fabric SDK is faked. @@ -215,8 +216,10 @@ def __init__(self, **kwargs: Any) -> None: trial = result.trials[0] assert trial.status == "completed" # The trajectory is exposed under the standard trace key, as an existing ATIF file. + assert trial.evidence is not None trace = trial.evidence.descriptors[EVIDENCE_TRACE] assert trace.format == EVIDENCE_FORMAT_ATIF + assert trace.ref is not None assert Path(trace.ref).exists() # The metric received the evidence and scored from the trajectory content. scores = [s for s in result.scores if s.metric_type == "has-trajectory"] @@ -242,13 +245,13 @@ def _codex_adapter_installed() -> bool: # No NeMo-Fabric checkout in the gate: the adapter registry resolves from the installed wheels -# (/share/nemo-fabric/adapters), so `uv sync --extra fabric` is enough. +# (/share/nemo-fabric/adapters), so the package-scoped `fabric` extra is enough. _LIVE_READY = bool(shutil.which("codex") and shutil.which("nemo-relay") and _codex_adapter_installed()) _LIVE_MODEL = os.environ.get("NEMO_FABRIC_LIVE_MODEL", "gpt-5.6-terra") requires_live_fabric = pytest.mark.skipif( not _LIVE_READY, reason=( - "needs the harness adapters (uv sync --extra fabric) + the nemo-relay gateway " + "needs the harness adapters (uv sync --package nemo-evaluator-sdk --extra fabric) + the nemo-relay gateway " "(script/dev-install-fabric.sh) + codex on PATH" ), ) @@ -263,9 +266,15 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None "harness": { "adapter_id": "nvidia.fabric.codex", "resolution": "preinstalled", - "settings": {"sandbox": "workspace-write", "skip_git_repo_check": True, "timeout_seconds": 180}, + "settings": {"sandbox": "workspace-write"}, + }, + "runtime": { + "mode": "oneshot", + "transport": "cli", + "input_schema": "text", + "output_schema": "message", + "timeout_seconds": 180, }, - "runtime": {"mode": "oneshot", "transport": "cli", "input_schema": "text", "output_schema": "message"}, "environment": {"provider": "local", "workspace": str(tmp_path / "ws")}, # Fabric's codex adapter requires an explicit model provider — it does not fall back to the # Codex CLI's own configured default, and starting without one fails the adapter lifecycle @@ -288,8 +297,10 @@ def test_fabric_codex_live_eval_captures_atif_trajectory(tmp_path: Path) -> None trial = result.trials[0] assert trial.status == "completed", trial.metadata + assert trial.evidence is not None trace = trial.evidence.descriptors[EVIDENCE_TRACE] assert trace.format == EVIDENCE_FORMAT_ATIF + assert trace.ref is not None atif = Path(trace.ref) assert atif.exists() and atif.stat().st_size > 0 assert "steps" in json.loads(atif.read_text(encoding="utf-8")) diff --git a/plugins/nemo-evaluator/README.md b/plugins/nemo-evaluator/README.md index bc79b6da31..334c55854d 100644 --- a/plugins/nemo-evaluator/README.md +++ b/plugins/nemo-evaluator/README.md @@ -1,73 +1,63 @@ # NeMo Evaluator Plugin -A NeMo Platform plugin that brings Evaluator SDK metric execution into the -platform. +The Evaluator plugin connects the NeMo Evaluator SDK to NeMo Platform. It +provides: -The plugin exposes an `evaluator` service, CLI commands under `nemo evaluator`, -an SDK accessor on `NeMoPlatform.evaluator`, and an `evaluator.run/evaluator.submit` for -local plugin runs and durable platform submissions. - -## What it provides - -- **CLI** commands for plugin status, job schema inspection, local runs, and +- **CLI** `nemo evaluator` commands for plugin status, job schema inspection, local runs, and job submissions. -- **Service** routes for evaluator job management. +- **Service** routes for evaluator job management: `plugins/nemo-evaluator/src/nemo_evaluator/service.py`. - **SDK accessor** at `client.evaluator` for status checks, local runs, job submission, status polling, result retrieval, and artifact download. - **Evaluator job** support for inline SDK metric specs, inline rows, and Fileset-backed datasets. + - Dataset-driven `evaluator.evaluate` jobs. + - Task-driven `evaluator.agent-evaluate` jobs. - **Docs and skills** that are published through the plugin entry points for evaluator-specific reference and troubleshooting. -## Installation (developer) - -Prerequisites: +## Developer setup -- Python and `uv` are available. -- Commands run from the repo root. -- `NVIDIA_API_KEY` is exported when running online or model-backed metrics. - -This plugin is a `uv` workspace member. From the repo root: +This plugin is a `uv` workspace member. From the repository root: ```bash -uv sync +# The `make bootstrap` target creates the Python environment, syncs Python dependencies, builds Studio assets, and installs local plugins. +make bootstrap +source .venv/bin/activate ``` -For local platform testing, start the platform after syncing: +Verify the installation: ```bash -nemo services run +nemo --help ``` -The root workspace also includes this plugin in the enabled plugin set, so the -`nemo evaluator` CLI group should be available in the synced environment. - -## CLI quickstart - -Check that the plugin is installed: +Check the plugin status: ```bash -nemo evaluator info +uv run nemo evaluator info ``` -Inspect the registered job contract: +Follow the repository `SETUP.md` for detailed setup instructions and starting local NeMo Platform services. -```bash -nemo evaluator evaluate explain -``` +## Dataset-Driven vs. Task-Driven evaluation +Review the [Evaluator documentation](https://docs.nvidia.com/nemo-platform/documentation/evaluate-models#two-shapes-of-evaluation) for a detailed explanation of the difference between dataset-driven and task-driven evaluation. -Run a minimal exact-match metric from the bundled example spec: +## Dataset-Driven evaluation + +### CLI Commands +Inspect the current schema and run the checked offline example: ```bash -nemo evaluator evaluate run \ - --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json +uv run nemo evaluator evaluate explain +uv run nemo evaluator evaluate run \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json ``` -Submit the same spec as a platform durable job: +Submit the same spec as a durable job: ```bash -nemo evaluator evaluate submit \ - --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_metric.json +uv run nemo evaluator evaluate submit \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json ``` The submit response includes a generated job name, for example `nemo-evaluator-zlhn1ecd`. Wait for the job to complete, then list and download its results: @@ -79,64 +69,130 @@ nemo jobs results download aggregate-scores --job --output-file aggre nemo jobs results download row-scores --job --output-file row-scores.jsonl ``` -## Python SDK quickstart +See also the checked LLM-judge spec example in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json`. -Use the mounted platform SDK accessor, `client.evaluator`: +### Plugin SDK Execution +Use the mounted SDK resource for local and durable execution: ```python from nemo_evaluator_sdk import ExactMatchMetric, RunConfig from nemo_platform import NeMoPlatform - client = NeMoPlatform(base_url="http://localhost:8080", workspace="default") -status = client.evaluator.plugin_status() - metric = ExactMatchMetric( reference="{{item.expected}}", - candidate="{{item.model_output}}", + candidate="{{item.output}}", ) dataset = [ - {"expected": "blue", "model_output": "Blue"}, - {"expected": "Jupiter", "model_output": "Saturn"}, + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, ] -local_result = client.evaluator.run( - metric=metric, - dataset=dataset, - config=RunConfig(parallelism=2), -) - job = client.evaluator.submit( metric=metric, dataset=dataset, config=RunConfig(parallelism=2), ) + job.wait_until_done() -submitted_result = job.get_result() -artifact_dir = job.download_artifacts(path="evaluation-artifacts") +remote_result = job.get_result() +artifact_dir = job.download_artifacts("evaluation-artifacts") ``` +`submit` returns an `EvaluatorJobResource`. Always call +`wait_until_done()` before retrieving result artifacts. + +## Task-Driven Agent evaluation + +### CLI Commands + +#### Durable job + +Inspect the task-driven job schema: + +```bash +uv run nemo evaluator agent-evaluate explain +``` + +The checked spec gives Fabric one task and scores the runner's final response +with exact match. Submit it as a durable platform job: + +```bash +uv run nemo evaluator agent-evaluate submit \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json +``` + +Replace `/` and ensure the job environment includes the Fabric +Codex adapter, Codex CLI, and its provider credentials. Set +`capture_trajectory` to `true` only when NeMo Relay is also available. +For repository setup, follow +[Prepare Fabric in a repository checkout](../../skills/nemo-evaluator-plugin/SKILL.md#prepare-fabric-in-a-repository-checkout). + +### SDK Execution +Plugin SDK execution is not supported for task-driven evaluation. Use the standalone Python SDK instead, which is available for local execution. + +#### Standalone SDK + +For an in-process agent callable, pass a direct `AgentTaskRunner` to the +standalone SDK: + +```python +from nemo_evaluator_sdk import ExactMatchMetric +from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator +from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import CallableAgentTaskRunner +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask + + +async def answer(task: AgentEvalTask) -> str: + return "Paris" + + +task = AgentEvalTask( + id="capital-france", + intent="Name the capital of France.", + inputs={"instruction": "What is the capital of France?"}, + reference={"expected": "Paris"}, + metrics=[ + ExactMatchMetric( + reference="{{reference.expected}}", + candidate="{{sample.output_text}}", + ) + ], +) +result = AgentEvaluator().run_sync( + tasks=[task], + target=CallableAgentTaskRunner(answer), +) +print(result.summary) +``` + +See the [agent-evaluation reference](../../skills/nemo-evaluator-plugin/references/agent-evaluation.md) +for tasksets, other durable targets, and precomputed trials. + +## Stored resources -## Local and remote inputs +The SDK namespace includes: -### Dataset support +- `client.evaluator.metrics` +- `client.evaluator.tasks` +- `client.evaluator.tasksets` +- `client.evaluator.eval_results` +- `client.evaluator.agent_eval_results` -- Local runs support local dataset paths, inline rows, and Fileset references. -- Jobs support inline rows and Fileset references. +Metrics, tasks, and tasksets support create, retrieve, list, and delete. Result +resources support retrieve, list, and delete. -### Model/Agent Auth +## Authentication -For online evaluation or LLM-as-judge evaluations, authentication depends on the -execution mode: +- Local model-backed evaluation resolves `api_key_secret` as a local + environment-variable name, such as `NVIDIA_API_KEY`.. +- Durable platformjobs resolve it as a NeMo Platform secret in the target workspace. -- Local `nemo evaluator evaluate run` resolves `api_key_secret` as a local - environment variable name, such as `NVIDIA_API_KEY`. -- Remote `nemo evaluator evaluate submit` resolves `api_key_secret` as a NeMo - Platform secret in the target workspace. +Never place a credential value in a spec or log. -## Next steps +## References -- [Evaluator plugin reference](src/nemo_evaluator/docs/index.md) -- [Evaluator platform docs](../../docs/evaluator/index.md) -- [Evaluator plugin skill](src/nemo_evaluator/skills/evaluator-plugin/SKILL.md) +- [Plugin reference](src/nemo_evaluator/docs/index.md) +- [Evaluator documentation](https://docs.nvidia.com/nemo-platform/documentation/evaluate-models) +- [Canonical evaluator skill](../../skills/nemo-evaluator-plugin/SKILL.md) - [Evaluator API auth](../../skills/nemo-evaluator-plugin/references/api-auth.md) -- [Evaluation troubleshooting](../../skills/nemo-evaluator-plugin/references/troubleshooting.md) +- [Troubleshooting](../../skills/nemo-evaluator-plugin/references/troubleshooting.md) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py index d74638d0e9..5ff1e28b43 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/cli.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/cli.py @@ -102,7 +102,7 @@ def info() -> None: "plugin": self.name, "status": "ready", "service": "/apis/evaluator/v1/healthz", - "jobs": ["evaluator.evaluate"], + "jobs": ["evaluator.evaluate", "evaluator.agent-evaluate"], "sdk": "nemo_evaluator_sdk.Evaluator", } ) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md b/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md index c9de7e9808..1dba88f560 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md +++ b/plugins/nemo-evaluator/src/nemo_evaluator/docs/index.md @@ -1,93 +1,156 @@ # Evaluator Plugin Reference -The evaluator plugin is a first-party for evaluator functionality. It keeps the plugin identity separate from the legacy `/apis/evaluation` service while proving the basic surfaces needed for SDK-backed jobs. +The Evaluator plugin connects the NeMo Evaluator SDK to NeMo Platform. Its +command group is `nemo evaluator`; the legacy generated `nemo evaluation` +surface is not used for new workflows. -## Registered Surfaces +## Registered interfaces -| Surface | Entry point | Current behavior | -|---|---|---| -| CLI | `nemo.cli:evaluator` | Adds `nemo evaluator info` and hosts evaluator job commands. | -| Service | `nemo.services:evaluator` | `jobs`, `healthz` paths. | -| SDK | `nemo.sdk:evaluator` | Adds `client.evaluator.plugin_status() and run(), submit() interfaces`. | -| Job | `nemo.jobs:evaluator.evaluate` | Backs local `run` through in-process execution and `submit` through durable platform job submission. | -| Docs | `nemo.docs:evaluator` | Publishes this reference page. | -| Skills | `nemo.skills:evaluator` | Publishes the evaluator plugin development skill. | +| Surface | Entry point | Behavior | +| --- | --- | --- | +| CLI | `nemo.cli:evaluator` | Plugin status, metric discovery, job schema inspection, local runs, and durable submissions | +| Service | `nemo.services:evaluator` | Health, job, stored-resource, and result routes | +| SDK | `nemo.sdk:evaluator` | `client.evaluator` execution, job lifecycle, stored resources, and result indexes | +| Dataset job | `nemo.jobs:evaluator.evaluate` | Scores inline or Fileset-backed datasets | +| Agent job | `nemo.jobs:evaluator.agent-evaluate` | Runs or rescores task-driven agent trials | +| Docs | `nemo.docs:evaluator` | Publishes this reference | +| Skill | `nemo.skills:evaluator` | Publishes the evaluator agent skill | -## Current Job +Confirm the installed surfaces: -`evaluator.evaluate` is a `NemoJob` that calls `packages/nemo_evaluator_sdk.Evaluator` directly. It currently supports inline datasets with `exact-match` and `string-check` metric configs. +```bash +uv run nemo evaluator info +uv run nemo evaluator evaluate explain +uv run nemo evaluator agent-evaluate explain +``` +When the plugin is installed outside this repository, omit the `uv run` prefix. -## CLI Examples +## Dataset-driven evaluation -### Prerequisite for online evaluation and model-backed metrics +`evaluator.evaluate` accepts: -#### Set API key +- `metrics`: one or more inline metric bundles or stored metric references. +- `dataset`: inline rows or a `FilesetRef`. +- `params`: offline, online, or online-model run configuration. +- `target`: an optional `Model` or `Agent`. +- `prompt_template`: the online generation prompt. +- `field_mapping`: canonical evaluator fields mapped to dataset columns. -Online evaluation examples call [NVIDIA-hosted models](https://build.nvidia.com/models) through the API key referenced by each spec's `api_key_secret`. +Run the checked offline pass/fail example: -To generate an API key on the NVIDIA Build hub: +```bash +uv run nemo evaluator evaluate run \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +``` -1. Sign in to your NVIDIA account at . -2. Open [API Keys](https://build.nvidia.com/settings/api-keys) and click **Generate API Key**. -3. Export the key before running the CLI: `export NVIDIA_API_KEY=`. +Submit the same shape as a durable job: -#### How to use API key +```bash +uv run nemo evaluator evaluate submit \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +``` -For evaluator API key auth, see [Evaluator API Auth](../../../../../skills/nemo-evaluator-plugin/references/api-auth.md) +The checked LLM-judge example is +`skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json`. -### Examples +### Python SDK -Check that the plugin is installed and reports the registered job key: +`client.evaluator.run` executes one runtime metric locally and returns an +`EvaluationResult`. `submit` creates a durable job resource: -```bash -nemo evaluator info -``` +```python +from nemo_evaluator_sdk import ExactMatchMetric +from nemo_platform import NeMoPlatform + +client = NeMoPlatform(base_url="http://localhost:8080", workspace="default") +metric = ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", +) +dataset = [ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, +] -Inspect the generated job metadata: +local_result = client.evaluator.run(metric=metric, dataset=dataset) -```bash -nemo evaluator evaluate explain +job = client.evaluator.submit(metric=metric, dataset=dataset) +job.wait_until_done() +remote_result = job.get_result() +artifact_dir = job.download_artifacts("evaluation-artifacts") ``` -Run an inline exact-match metric: +Always wait for terminal completion before retrieving results. Metric progress +can reach 100 percent before the platform finalizes result artifacts. -```bash -nemo evaluator evaluate run --spec '{"metric":{"type":"exact-match","reference":"{{item.expected}}","candidate":"{{item.model_output}}"},"dataset":[{"expected":"blue","model_output":"Blue"},{"expected":"Jupiter","model_output":"Saturn"}],"params":{"parallelism":2}}' -``` +Local dataset inputs may also be paths. Local and durable jobs accept +`FilesetRef`. Durable submission additionally accepts a `ModelRef`; local +execution requires a concrete `Model`. -Run an online llm-as-judge metric from a spec file (requires `NVIDIA_API_KEY`, see the [prerequisite](#prerequisite-for-online-evaluation-and-model-backed-metrics) above): +## Task-driven agent evaluation -```bash -nemo evaluator evaluate run --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/llm_as_judge.json -``` +`evaluator.agent-evaluate` accepts inline tasks or a stored `TasksetRef`. Each +task has its own metrics. Provide exactly one of: -Run a benchmark metric from spec file example: +- `target` to generate trials. +- `trials` to rescore precomputed trials. -```bash -nemo evaluator evaluate run --spec-file plugins/nemo-evaluator/src/nemo_evaluator/docs/data/exact_match_benchmark.json -``` +Durable target variants are: -## Python Examples +- `ModelTarget` +- `AgentTarget` +- `CodexRunnerTarget` +- `FabricRunnerTarget` +- `HarborRunnerTarget` -Read the plugin service status through the platform SDK namespace: +The standalone SDK additionally accepts a direct `AgentTaskRunner`: ```python -from nemo_platform import NeMoPlatform - -client = NeMoPlatform(base_url="http://localhost:8080") -status = client.evaluator.plugin_status() +result = await AgentEvaluator().run(tasks=tasks, target=runner) ``` -Use the evaluator SDK directly, matching the job's current execution path: +Use `max_concurrent_tasks` for task-level concurrency. Target-specific fields +control inference or runner concurrency. `fail_fast` stops on the first scoring +failure, and `benchmark` records run metadata. -```python -from nemo_evaluator_sdk import Evaluator -from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +Harbor tasks must preserve `harbor_dataset_path` metadata from task discovery. +The runtime also requires Harbor, Docker, and any selected agent dependencies. -metric = ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}") -result = Evaluator().run_sync( - metrics=metric, - dataset=[{"expected": "blue", "model_output": "Blue"}], -) -``` +Agent-evaluation jobs write trials, scores, summaries, evidence, and report +artifacts, and create a queryable record under +`client.evaluator.agent_eval_results`. + +## Stored resources + +`client.evaluator` exposes immutable definitions and persisted result indexes: + +| Resource | Operations | Notable filters | +| --- | --- | --- | +| `metrics` | create, retrieve, list, delete | `metric_type`, `include_derived` | +| `tasks` | create, retrieve, list, delete | pagination and sort | +| `tasksets` | create, retrieve, list, delete | pagination and sort | +| `eval_results` | retrieve, list, delete | `job_id`, target, `dataset_ref` | +| `agent_eval_results` | retrieve, list, delete | `job_id`, target | + +Metrics, tasks, and tasksets have no update operation. Delete and recreate them +or use a new versioned name. + +Stored tasks keep metric references but do not keep grader-only task +`reference`; use inline agent-evaluation tasks when a metric needs held-out +per-task data. + +## Authentication and metric packaging + +For local execution, `api_key_secret` names a local environment variable. For +durable submission, it names a NeMo Platform secret in the target workspace. +Inspect a remote 409 response body because it can report a missing platform +secret rather than a duplicate job. + +Built-in metrics use declarative inline bundles by default. Submitting custom +Python metrics requires an explicit `HybridMetricBundlePackager` or +`CloudpickleMetricBundlePackager`. + +See the canonical +[Evaluator plugin skill](../../../../../skills/nemo-evaluator-plugin/SKILL.md) +for the execution workflow and focused references. diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index 147fbffd83..a321deb950 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -6,7 +6,6 @@ from __future__ import annotations import json -from collections.abc import Callable from pathlib import Path from types import SimpleNamespace from typing import Any, Literal, cast @@ -27,6 +26,7 @@ EvaluateJob, EvaluateSpec, ) +from nemo_evaluator.jobs.metric_resolution import to_runtime_bundle from nemo_evaluator.resolvers import PlatformModelResolver, _parse_required_workspace_name from nemo_evaluator.shared.metric_bundles.bundles import ( MetricBundle, @@ -44,7 +44,6 @@ from nemo_evaluator_sdk.metrics.f1 import F1Metric from nemo_evaluator_sdk.metrics.llm_judge import LLMJudgeMetric from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult -from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric from nemo_evaluator_sdk.values import ( Agent, AggregatedMetricResult, @@ -69,9 +68,7 @@ from pytest_mock import MockerFixture from typer.testing import CliRunner -ExampleSpecBuilder = Callable[[], dict[str, Any]] EXAMPLE_SPEC_PATHS = ( - Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json"), Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json"), Path("skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json"), ) @@ -98,152 +95,6 @@ def _repo_root() -> Path: return Path(__file__).resolve().parents[3] -def _generated_exact_match_metric_spec() -> dict[str, Any]: - return { - "metrics": [ - _bundle_payload(ExactMatchMetric(reference="{{item.expected}}", candidate="{{item.model_output}}")), - ], - "dataset": [ - {"expected": "blue", "model_output": "Blue"}, - {"expected": "Jupiter", "model_output": "Saturn"}, - ], - "params": {"parallelism": 2}, - } - - -def _generated_exact_match_benchmark_spec() -> dict[str, Any]: - return { - "metrics": [ - _bundle_payload(ExactMatchMetric(reference="{{item.reference}}")), - _bundle_payload( - StringCheckMetric( - operation="contains", - left_template="{{sample.output_text}}", - right_template="{{item.required_phrase}}", - ) - ), - ], - "dataset": [ - { - "prompt": "Return exactly this word with no punctuation: Paris", - "reference": "Paris", - "required_phrase": "Paris", - }, - { - "note": ( - "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are " - "'London' so both metrics should report a miss." - ), - "prompt": "Return exactly this word with no punctuation: Oslo", - "reference": "London", - "required_phrase": "London", - }, - ], - "params": { - "parallelism": 4, - "limit_samples": 2, - "ignore_request_failure": False, - "request_timeout": 60, - "max_retries": 3, - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim", - }, - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "{{item.prompt}}", - } - ] - }, - } - - -def _generated_llm_as_judge_spec() -> dict[str, Any]: - return { - "metrics": [ - _bundle_payload( - LLMJudgeMetric( - model=Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="nvidia/nemotron-3-super-120b-a12b", - api_key_secret=SecretRef(root="NVIDIA_API_KEY"), - format="nim", - ), - scores=[ - RangeScore( - name="helpfulness", - description="How well does the response help the user?", - minimum=0, - maximum=4, - parser=JSONScoreParser(json_path="helpfulness"), - ) - ], - prompt_template={ - "messages": [ - { - "role": "system", - "content": ( - "You are an evaluator. Rate the response's helpfulness from 0-4. " - "Return only a JSON object with this shape: " - '{"helpfulness": }.' - ), - }, - { - "role": "user", - "content": ( - "User prompt: {{item.input}}\n\n" - "Assistant response: " - "{{sample.output_text | default(item.output)}}\n\n" - "Rate this response." - ), - }, - ] - }, - ) - ) - ], - "dataset": [ - {"input": "What is the capital of France?"}, - {"input": "How do I make scrambled eggs?"}, - ], - "params": { - "parallelism": 2, - "limit_samples": 2, - "request_timeout": 120, - "max_retries": 3, - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim", - }, - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "{{item.input}}", - } - ] - }, - } - - -def _example_spec_builders() -> dict[Path, ExampleSpecBuilder]: - return { - Path( - "skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json" - ): _generated_exact_match_benchmark_spec, - Path("skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json"): _generated_exact_match_metric_spec, - Path("skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json"): _generated_llm_as_judge_spec, - } - - def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None: step = job_spec.steps[0] container = cast(Any, step.executor).container @@ -414,7 +265,7 @@ def _llm_judge_ref_metric() -> LLMJudgeMetric: "spec_path", EXAMPLE_SPEC_PATHS, ) -def test_checked_in_example_spec_uses_metric_bundle_shape(spec_path: Path) -> None: +def test_checked_in_example_spec_uses_inline_metric_bundle_shape(spec_path: Path) -> None: payload = json.loads((_repo_root() / spec_path).read_text(encoding="utf-8")) spec = EvaluateInputSpec.model_validate(payload) @@ -423,18 +274,17 @@ def test_checked_in_example_spec_uses_metric_bundle_shape(spec_path: Path) -> No assert len(spec.metrics) >= 1 for metric_payload in payload["metrics"]: bundle = MetricBundle.model_validate(metric_payload) - # Static cloudpickle fixtures are Python-minor-version specific, so - # this test validates the checked-in bundle envelope without hydrating. - assert bundle.payload.kind == "cloudpickle" + assert bundle.payload.kind == "inline" assert bundle.metric_type == metric_payload["metric_type"] + assert unbundle_metric(bundle).type == bundle.metric_type @pytest.mark.parametrize( "spec_path", EXAMPLE_SPEC_PATHS, ) -def test_generated_example_spec_compiles_with_runtime_cloudpickle(spec_path: Path) -> None: - payload = _example_spec_builders()[spec_path]() +def test_checked_in_example_spec_compiles(spec_path: Path) -> None: + payload = json.loads((_repo_root() / spec_path).read_text(encoding="utf-8")) spec = EvaluateSpec.model_validate(payload) compiled = compile_evaluate_job(spec) @@ -498,7 +348,7 @@ def test_cli_explain_uses_registered_evaluator_job_key() -> None: assert payload["spec_schema"]["title"] == "EvaluateSpec" -def test_cli_info_reports_registered_evaluator_job_key() -> None: +def test_cli_info_reports_registered_evaluator_job_keys() -> None: app = EvaluatorPluginCLI().get_cli() add_job_commands(app, {"evaluator.evaluate": EvaluateJob}) @@ -506,7 +356,7 @@ def test_cli_info_reports_registered_evaluator_job_key() -> None: assert result.exit_code == 0 payload = json.loads(result.output) - assert payload["jobs"] == ["evaluator.evaluate"] + assert payload["jobs"] == ["evaluator.evaluate", "evaluator.agent-evaluate"] def test_cli_metric_types_reports_sdk_metric_union_types() -> None: @@ -745,7 +595,7 @@ async def test_evaluate_job_to_spec_resolves_bundled_metric_model_refs_before_co is_local=False, ) assert isinstance(canonical, EvaluateSpec) - canonical_metric = unbundle_metric(canonical.metrics[0]) + canonical_metric = unbundle_metric(to_runtime_bundle(canonical.metrics[0])) assert isinstance(canonical_metric, LLMJudgeMetric) assert isinstance(canonical_metric.model, Model) assert canonical_metric.model.name == "judge" @@ -799,7 +649,7 @@ async def test_evaluate_job_to_spec_preserves_metric_without_model_refs() -> Non ) assert isinstance(canonical, EvaluateSpec) - metric = unbundle_metric(canonical.metrics[0]) + metric = unbundle_metric(to_runtime_bundle(canonical.metrics[0])) assert isinstance(metric, LLMJudgeMetric) assert metric.prompt_template is None diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py new file mode 100644 index 0000000000..f9fffb4aee --- /dev/null +++ b/plugins/nemo-evaluator/tests/test_skill_examples.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Focused validation for the canonical Evaluator plugin skill examples.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +from nemo_evaluator.api.schemas import TasksetRef +from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec, CodexRunnerTarget, FabricRunnerTarget +from nemo_evaluator.jobs.evaluate import EvaluateInputSpec +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric, unbundle_metric +from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager +from nemo_evaluator_sdk import ExactMatchMetric, LLMJudgeMetric, Model +from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator +from nemo_evaluator_sdk.agent_eval.persistence import read_trials +from nemo_evaluator_sdk.agent_eval.scores import AgentEvalScoreStatus +from nemo_evaluator_sdk.agent_eval.tasks import AgentEvalTask +from nemo_evaluator_sdk.agent_eval.trials import AgentEvalTrial, AgentEvalTrialStatus, AgentOutput + + +def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def _load_module(relative_path: str, name: str) -> ModuleType: + path = _repo_root() / relative_path + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _fenced_block_containing(markdown: str, *, language: str, needle: str) -> str: + marker = f"```{language}\n" + for remainder in markdown.split(marker)[1:]: + source = remainder.split("```", 1)[0] + if needle in source: + return source + raise ValueError(f"no {language} block contains {needle!r}") + + +def test_generated_skill_specs_are_current_and_inline() -> None: + generator = _load_module( + "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py", + "nemo_evaluator_skill_spec_generator", + ) + + assert generator.check_specs() == 0 + for payload in generator.generated_specs().values(): + spec = EvaluateInputSpec.model_validate(payload) + assert spec.metrics + for metric in spec.metrics: + bundle = MetricBundle.model_validate(metric.model_dump(mode="json")) + assert bundle.payload.kind == "inline" + + +def test_generated_llm_judge_spec_uses_local_environment_secret() -> None: + generator = _load_module( + "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py", + "nemo_evaluator_skill_secret_generator", + ) + + payload = generator.build_llm_as_judge_spec() + assert payload["target"]["api_key_secret"] == "NVIDIA_API_KEY" + + bundle = MetricBundle.model_validate(payload["metrics"][0]) + assert bundle.secrets["NVIDIA_API_KEY"].root == "NVIDIA_API_KEY" + + judge = unbundle_metric(bundle) + assert isinstance(judge, LLMJudgeMetric) + assert isinstance(judge.model, Model) + assert judge.model.api_key_secret is not None + assert judge.model.api_key_secret.root == "NVIDIA_API_KEY" + + +def test_local_llm_judge_spec_guides_platform_secret_remap() -> None: + root = _repo_root() / "skills/nemo-evaluator-plugin/references" + auth = (root / "api-auth.md").read_text(encoding="utf-8") + execution = (root / "execution.md").read_text(encoding="utf-8") + normalized_auth = " ".join(auth.split()) + + assert ".target.api_key_secret = $platform_secret" in auth + assert ".metrics[0].secrets.NVIDIA_API_KEY = $platform_secret" in auth + assert "Do not edit `metrics[*].payload`" in normalized_auth + assert "api-auth.md#adapt-the-local-first-spec-for-platform-submission" in execution + assert "--spec-file llm_as_judge.platform.json" in execution + + +def test_skill_python_examples_import_and_build_agent_spec() -> None: + examples = _load_module( + "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py", + "nemo_evaluator_skill_examples", + ) + metric_bundle = bundle_metric( + examples.capital_france_metric(), + InlineMetricBundlePackager(), + ).model_dump(mode="json") + + spec = AgentEvalInputSpec.model_validate(examples.build_agent_eval_spec(metric_bundle)) + + assert not isinstance(spec.tasks, TasksetRef) + assert len(spec.tasks) == 1 + assert isinstance(spec.target, CodexRunnerTarget) + + +def test_skill_standalone_example_scores_pass_and_failure() -> None: + examples = _load_module( + "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py", + "nemo_evaluator_skill_standalone_example", + ) + + result = examples.evaluate_standalone() + + assert len(result.row_scores) == 2 + assert result.aggregate_scores.scores[0].mean == 0.5 + + +def test_skill_agent_metric_scores_precomputed_output() -> None: + examples = _load_module( + "skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py", + "nemo_evaluator_skill_agent_metric", + ) + task = AgentEvalTask( + id="capital-france", + intent="Name the capital of France.", + inputs={"instruction": "What is the capital of France?"}, + metrics=[examples.capital_france_metric()], + ) + trial = AgentEvalTrial( + id="trial-1", + task_id=task.id, + status=AgentEvalTrialStatus.COMPLETED, + output=AgentOutput(output_text="Paris"), + ) + + result = AgentEvaluator().run_sync(tasks=[task], trials=[trial]) + + assert result.scores[0].status is AgentEvalScoreStatus.COMPLETED + assert result.scores[0].outputs[0].value == 1.0 + + +def test_checked_durable_fabric_job_is_a_valid_agent_eval_spec() -> None: + generator = _load_module( + "skills/nemo-evaluator-plugin/scripts/generate_example_specs.py", + "nemo_evaluator_skill_agent_spec_generator", + ) + path = _repo_root() / "skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json" + generated = generator.generated_agent_specs() + + assert generated[path] == json.loads(path.read_text(encoding="utf-8")) + spec = AgentEvalInputSpec.model_validate(generated[path]) + + assert isinstance(spec.tasks, list) + metric = unbundle_metric(MetricBundle.model_validate(spec.tasks[0].metrics[0].model_dump(mode="json"))) + assert isinstance(metric, ExactMatchMetric) + assert isinstance(spec.target, FabricRunnerTarget) + assert spec.target.capture_trajectory is False + assert spec.max_concurrent_tasks == 1 + + +def test_readme_standalone_direct_runner_scores_task(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + readme = (_repo_root() / "plugins/nemo-evaluator/README.md").read_text(encoding="utf-8") + source = _fenced_block_containing(readme, language="python", needle="CallableAgentTaskRunner(answer)") + namespace: dict[str, Any] = {} + monkeypatch.chdir(tmp_path) + + exec(compile(source, "plugins/nemo-evaluator/README.md", "exec"), namespace) + + result = namespace["result"] + assert result.trials[0].output.output_text == "Paris" + assert result.scores[0].outputs[0].value == 1.0 + + +def test_skill_points_to_working_repository_fabric_installer() -> None: + root = _repo_root() + skill = (root / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8") + + assert "uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact" in skill + assert "script/dev-install-fabric.sh" in skill + assert (root / "script/dev-install-fabric.sh").is_file() + + +def test_skill_explains_cli_discovery_commands() -> None: + skill = (_repo_root() / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8") + + assert "# confirms plugin readiness and lists the registered evaluator jobs." in skill + assert "# lists available metric names; add a metric name to print its schema." in skill + assert "# next two commands print the dataset-driven and task-driven job input and" in skill + assert "# output schemas - can be very large" in skill + + +def test_skill_routes_dataset_examples_to_references() -> None: + skill = (_repo_root() / "skills/nemo-evaluator-plugin/SKILL.md").read_text(encoding="utf-8") + + assert "references/execution.md#validate-standalone-then-submit-to-the-platform" in skill + assert "references/execution.md#getting-job-results" in skill + assert "references/resources.md#store-a-metric-task-and-taskset" in skill + assert "references/resources.md#query-persisted-results" in skill + assert "result = Evaluator().run_sync(" not in skill + assert "job = client.evaluator.submit(" not in skill + + +def test_skill_links_to_evaluation_shape_guidance() -> None: + root = _repo_root() / "skills/nemo-evaluator-plugin" + skill = (root / "SKILL.md").read_text(encoding="utf-8") + reference = root / "references/evaluation-shapes.md" + guidance = reference.read_text(encoding="utf-8") + + assert "references/evaluation-shapes.md#dataset-driven-evaluation" in skill + assert "references/evaluation-shapes.md#task-driven-evaluation" in skill + assert "[SDK Execution](execution.md)" in guidance + assert "[Agent Evaluation](agent-evaluation.md)" in guidance + + +def test_skill_names_concrete_standalone_agent_targets() -> None: + root = _repo_root() / "skills/nemo-evaluator-plugin" + skill = (root / "SKILL.md").read_text(encoding="utf-8") + reference = (root / "references/agent-evaluation.md").read_text(encoding="utf-8") + sections = ( + skill.split("**Standalone SDK evaluation**", 1)[1].split("**Platform job evaluation**", 1)[0], + reference.split("The standalone target union is:", 1)[1].split("For a minimal direct runner:", 1)[0], + ) + + for section in sections: + assert "`GenericAgent`" in section + assert "AgentTaskRunner" in section + assert "`Agent`" not in section + assert "NemoAgentToolkitAgent" not in section + + +def test_execution_pairs_python_examples_with_cli_when_supported() -> None: + reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/execution.md").read_text(encoding="utf-8") + + python_blocks = reference.count("```python") + cli_blocks = reference.count("```bash") + assert python_blocks + assert cli_blocks >= python_blocks + + submit_block = reference.split("**Platform Python SDK**", 1)[1].split("```python", 1)[1].split("```", 1)[0] + assert "metric=ExactMatchMetric(" in submit_block + assert "dataset=[" in submit_block + + +def test_metric_selection_points_to_metric_protocol() -> None: + reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/metric-selection.md").read_text( + encoding="utf-8" + ) + + assert "nemo_evaluator_sdk.metrics.protocol.Metric" in reference + assert "nemo_evaluator_sdk.values.protocol.Metric" not in reference + + +def test_multiple_metric_platform_submission_uses_cli() -> None: + reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/execution.md").read_text(encoding="utf-8") + section = reference.split("## Multiple metrics", 1)[1].split("## Package metrics safely", 1)[0] + + assert "Python SDK" not in section + assert "client.evaluator.submit" not in section + assert "nemo evaluator evaluate submit --spec-file multi-metric.json" in section + + +def test_resources_show_inline_task_before_held_out_reference_guidance() -> None: + reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/resources.md").read_text(encoding="utf-8") + + example_position = reference.index("inline_task = AgentEvalTaskInput(") + guidance_position = reference.index("Stored tasks keep metric references.") + assert example_position < guidance_position + assert 'reference={"expected": "Paris"}' in reference + + +def test_agent_evaluation_shows_how_to_retrieve_stored_trials() -> None: + reference = (_repo_root() / "skills/nemo-evaluator-plugin/references/agent-evaluation.md").read_text( + encoding="utf-8" + ) + + assert 'agent_eval_results.retrieve("")' in reference + assert "client.files.download(remote_path=stored.bundle_ref" in reference + assert 'read_trials("previous-run")' in reference + assert "nemo jobs results download agent-eval-results" in reference + assert callable(read_trials) + + +def test_authored_skill_guidance_uses_submit_for_plugin_jobs() -> None: + root = _repo_root() / "skills/nemo-evaluator-plugin" + authored_paths = [ + root / "SKILL.md", + *sorted((root / "references").glob("*.md")), + *sorted((root / "assets/examples").glob("*.py")), + ] + guidance = "\n".join(path.read_text(encoding="utf-8") for path in authored_paths) + + forbidden = ( + "client.evaluator.create(", + "client.evaluator.run(", + "nemo evaluator evaluate run", + "nemo evaluator agent-evaluate run", + "evaluate run/submit", + "agent-evaluate run/submit", + "local plugin", + ) + assert not any(term in guidance for term in forbidden) + + assert "Evaluator().run_sync(" in guidance + assert "AgentEvaluator().run(" in guidance + assert "client.evaluator.submit(" in guidance + assert "nemo evaluator evaluate submit" in guidance + assert "nemo evaluator agent-evaluate submit" in guidance diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh index d560455bcd..c008ec9dbf 100755 --- a/script/dev-install-fabric.sh +++ b/script/dev-install-fabric.sh @@ -5,21 +5,24 @@ # Dev-only: install the `nemo-relay` GATEWAY BINARY, the one Fabric eval dependency that cannot come # from a wheel. It is required for live ATIF trajectory capture on out-of-process harnesses (codex). # -# Everything else is in the lock — `uv sync --extra fabric` installs the nemo-fabric SDK, the -# codex/claude/deepagents adapters, and the nemo-relay Python bindings. The pip `nemo-relay` package -# is bindings-only (its wheel declares no console script and contains no executable), so the daemon is -# published solely as a GitHub release asset. +# Everything else is in the lock — +# `uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact` installs the nemo-fabric +# SDK, the codex/claude/hermes adapters, and the nemo-relay Python bindings without removing the +# existing workspace environment. The pip `nemo-relay` package is bindings-only (its wheel declares +# no console script and contains no executable), so the daemon is published solely as a GitHub +# release asset. # # The version defaults to the `nemo-relay` bindings installed in the venv, so the daemon and the # bindings cannot drift apart when the lock moves. # # To run against an unreleased Fabric instead of the locked wheels, install the checkout directly: # uv pip install --python .venv/bin/python "/path/to/NeMo-Fabric[codex,relay,runtime]" -# and `uv sync --extra fabric` to get back to the locked state. (That needs cargo — Fabric builds a -# Rust/pyo3 extension from source.) +# and use the package-scoped command above to get back to the locked state. (That needs cargo — +# Fabric builds a Rust/pyo3 extension from source.) # # A live codex run additionally needs the `codex` CLI + `codex login` auth. -# See plugins/nemo-evaluator/docs/design/fabric-runner-integration.md. +# See skills/nemo-evaluator-plugin/references/agent-evaluation.md and +# packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py. # # Usage: # script/dev-install-fabric.sh # version matching the installed bindings @@ -38,7 +41,8 @@ if [ -z "${NEMO_RELAY_VERSION:-}" ]; then bindings_version="$("$VENV_PY" -c 'import importlib.metadata as m; print(m.version("nemo-relay"))' 2>/dev/null || true)" if [ -z "$bindings_version" ]; then echo "nemo-relay is not installed in $VENV_PY, so the gateway version cannot be derived." >&2 - echo "Run 'uv sync --extra fabric' first, or pass NEMO_RELAY_VERSION= explicitly." >&2 + echo "Run 'uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact' first," >&2 + echo "or pass NEMO_RELAY_VERSION= explicitly." >&2 exit 1 fi NEMO_RELAY_VERSION="$(printf '%s' "$bindings_version" | sed -E 's/([0-9])(a|b|rc)\.?([0-9]+)$/\1-\2.\3/')" diff --git a/skills/nemo-evaluator-plugin/SKILL.md b/skills/nemo-evaluator-plugin/SKILL.md index e9966e7927..19643954a8 100644 --- a/skills/nemo-evaluator-plugin/SKILL.md +++ b/skills/nemo-evaluator-plugin/SKILL.md @@ -1,145 +1,122 @@ --- name: nemo-evaluator-plugin -description: Use when working on the Evaluator plugin CLI, jobs, SDK-backed specs, metric types, or plugin-owned Evaluator skills. -metadata: - owner: nemo-platform - maturity: active -license: Apache-2.0 +description: Use when evaluating models or agents with the NeMo Evaluator plugin; choosing deterministic, LLM, RAG, or agentic metrics; submitting evaluator jobs; managing stored metrics, tasks, tasksets, or results; or configuring agent-evaluate agent, Codex, Fabric, Harbor, and custom runner targets. --- # Evaluator Plugin -Use this skill for evaluation tasks against a running NeMo Platform server. The plugin-backed CLI interface is `nemo evaluator`; the legacy generated `nemo evaluation` API command group is not the target surface for new guidance. +The Plugin CLI entrypoint is `nemo evaluator`. -## CLI Interface - -### Prerequisites - -- all commands in this file assume that the shell's working dir is at the root of the Nvidia-NeMo/nemo-platform repo -- activate the Python virtual environment before invoking the `nemo` CLI: `source .venv/bin/activate` - -Check plugin status from the CLI: - -```bash -nemo evaluator info -``` - -## Metric Types - -### Explore Available Metrics - -To view available metric names, run: - -```bash -nemo evaluator metric-types -``` - -To view a specific metric schema, pass a metric name from the `metric_types` list above: - -```bash -nemo evaluator metric-types -``` - -Inspect all the registered metric schema contracts: - -```bash -nemo evaluator evaluate explain -``` - -> Note: use `nemo evaluator evaluate explain` as the source of truth for the current plugin input schema. It will return a large json schema response, so strongly prefer `nemo evaluator metric-types` when you only need metric names and corresponding schemas. +## Follow the evaluation loop -## Evaluation Spec +1. Clarify whether the input is [dataset-driven rows](references/evaluation-shapes.md#dataset-driven-evaluation) + or [task-driven agent work](references/evaluation-shapes.md#task-driven-evaluation). +2. Choose the simplest metric that measures the requested behavior. Prefer deterministic metrics when possible. +3. Build a tiny smoke case with one expected pass and one expected failure. +4. Validate metric behavior with the standalone SDK and inspect row-level output plus aggregates. +5. Fix field mappings, prompts, parsers, or task definitions before scaling. +6. Submit the platform job only after the input and scoring shape works. -Evaluation spec is a payload that is provided to CLI as an input to execute evaluation. +Read [Metric Selection](references/metric-selection.md) before choosing a +metric for a rubric, RAG workflow, or tool-calling evaluation. -At a high level, a spec describes: +## Choose the execution interface -- `metrics`: bundled Evaluator SDK metric configurations -- `dataset`: inline rows to evaluate or platform FilesetRef that contains the dataset -- `params`: optional Evaluator SDK execution parameters -- `target`: optional model or agent target for online evaluation +| Need | Interface | +| --- | --- | +| Fast metric iteration without NeMo Platform | `nemo_evaluator_sdk.Evaluator` | +| Dataset-driven platform job | `client.evaluator.submit(...)` or `nemo evaluator evaluate submit` | +| Multiple inline/stored metric refs in one job | `nemo evaluator evaluate submit` with an `EvaluateInputSpec` | +| Task-driven platform job | `nemo evaluator agent-evaluate submit` | +| Reusable platform definitions and result indexes | `client.evaluator.metrics`, `.tasks`, `.tasksets`, `.eval_results`, `.agent_eval_results` | -See the LLM-judge spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json). +- Read [SDK Execution](references/execution.md) for datasets, targets, +configuration, field mapping, job lifecycle, and custom metric packaging. +- Read [Stored Resources](references/resources.md) for persisted definitions and +result queries. -### Metric Bundle Payloads - -The checked-in [spec examples](./assets/specs) use bundled SDK metrics. The fields under `metrics[*].payload` are generated by `bundle_metric(metric, CloudpickleMetricBundlePackager())`. +## CLI Interface +### Prerequisites +All commands in this file assume that the shell's working dir is at the root of the Nvidia-NeMo/nemo-platform repo -To see the pattern for configuring a pre-defined SDK metric, for example `ExactMatchMetric`, and converting it into bundled metric JSON, inspect `build_metric_bundle_example()` in [generate_example_specs.py](./scripts/generate_example_specs.py) and run: +In a NeMo Platform repository checkout, run commands through the workspace: ```bash -uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py +# confirms plugin readiness and lists the registered evaluator jobs. +uv run nemo evaluator info +# lists available metric names; add a metric name to print its schema. +uv run nemo evaluator metric-types +# next two commands print the dataset-driven and task-driven job input and +# output schemas - can be very large, use with caution to avoid filling up the context window. +uv run nemo evaluator evaluate explain +uv run nemo evaluator agent-evaluate explain ``` -## Run Evaluations +When the skill and plugin are installed, use the installed `nemo` command +without assuming a repository root or manually activating `.venv`. -### Run Using File Spec Reference +Resolve bundled assets relative to this skill directory. In this repository the +canonical path is `skills/nemo-evaluator-plugin`; an installed skill may live +under a different skills root. -When using the `nemo evaluator evaluate run` command, results are saved into local temporary directories and the link is printed to stdout. -Prefer the `--spec-file` named argument over inline shell JSON because metric bundles include serialized payloads. -Examples of various specs are provided in the [assets/specs](./assets/specs/) directory. +## Dataset-driven evaluation examples -#### Evaluate using `exact-match` metric - -See the spec example at [assets/specs/exact_match_metric.json](./assets/specs/exact_match_metric.json). - -```bash -nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json -``` - -#### Evaluate using a benchmark metric set - -```bash -nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json -``` +- Follow [Validate standalone, then submit to the platform](references/execution.md#validate-standalone-then-submit-to-the-platform) + for the two-row pass/fail smoke test and its CLI submission. +- Follow [Map noncanonical fields](references/execution.md#map-noncanonical-fields) + when dataset columns need `field_mapping`. +- Follow [Getting job results](references/execution.md#getting-job-results) + for submission, terminal waiting, result retrieval, and artifact download. +- Follow [Store a metric, task, and taskset](references/resources.md#store-a-metric-task-and-taskset) + for reusable definitions, and [Query persisted results](references/resources.md#query-persisted-results) + for result lookup. -#### Evaluate using `LLM-Judge` metric +Always call `job.wait_until_done()` before retrieving results or downloading +artifacts; metric progress can reach 100 percent before the platform job is +terminal. -Uses an LLM to score responses. See the spec example at [assets/specs/llm_as_judge.json](./assets/specs/llm_as_judge.json). +## Task-driven agent evaluation examples +**Standalone SDK evaluation** +Use `AgentEvaluator().run(...)` for standalone task-driven SDK evaluation. Its +`target` can be a `Model`, a `GenericAgent`, or a direct `AgentTaskRunner`. -```bash -nemo evaluator evaluate run --spec-file skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json -``` +**Platform job evaluation** +Use the plugin `agent-evaluate submit` job for platform task evaluation. Its +target is a `ModelTarget`, `AgentTarget`, `CodexRunnerTarget`, +`FabricRunnerTarget`, or `HarborRunnerTarget`; alternatively provide +precomputed `trials`. Provide exactly one of `target` or `trials`. -### Run Evaluation As A Durable Job +Submission accepts inline tasks or a stored `TasksetRef`. Stored tasksets are +resolved in the target workspace. -Use the `nemo evaluator evaluate submit` command to create a durable evaluation job. The response of this command returns a job handler object instead of the evaluation result. +Read [Agent Evaluation](references/agent-evaluation.md) for inline tasks, +`TasksetRef`, concurrency, fail-fast behavior, result artifacts, and runner +configuration. -```bash -nemo evaluator evaluate submit \ - --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json -``` +### Prepare Fabric in a repository checkout -The submit response includes the generated job's `name` field, for example `nemo-evaluator-zlhn1ecd`. Wait for the job to complete, then list and download the job results. +Fabric runner examples and tests need the optional harness adapters and the +matching Relay gateway: ```bash -nemo jobs get-status -nemo jobs get -nemo jobs results list -nemo jobs results download aggregate-scores --job --output-file aggregate-scores.json -nemo jobs results download row-scores --job --output-file row-scores.jsonl +uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact +script/dev-install-fabric.sh ``` -## Python SDK Interface - -Evaluator Python SDK client is exposed as `evaluator` variable on `NeMoPlatform` instance: - -```python -from nemo_platform import NeMoPlatform - -platform_client = NeMoPlatform(base_url="http://localhost:8080") -status = platform_client.evaluator.plugin_status() -``` - -See examples of using the plugin SDK interface in [plugin_sdk_examples.py](./assets/examples/plugin_sdk_examples.py). - -## Security -Make sure not to print any secrets to stdout since this can be collected as logs - -## Additional Resources +The install script downloads the checksum-verified `nemo-relay` binary that +matches the locked Python bindings. Add its reported directory to `PATH`, then +use `uv run --frozen --no-sync ...` for Fabric checks so uv does not remove the +optional adapters. -For LLM-judge setup notes, see [LLM Judge Notes](references/llm-judge.md). +## Read specialized references -For evaluator API key auth, see [Evaluator API Auth](references/api-auth.md). +- Read [Evaluator API Auth](references/api-auth.md) before using a model, + agent, remote metric, or durable submission. +- Read [LLM Judge](references/llm-judge.md) before writing judge scores, + prompts, or parsers. +- Read [Troubleshooting](references/troubleshooting.md) when schema, + authentication, job, result, or runner behavior fails. -For local and cluster troubleshooting, see [Evaluation Troubleshooting](references/troubleshooting.md). +## Follow security best practices +Never print, serialize, or commit secret values. Store only environment-variable +names or platform secret references in specs and examples. diff --git a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py index e572ae5bb6..1634f9bc27 100644 --- a/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py +++ b/skills/nemo-evaluator-plugin/assets/examples/plugin_sdk_examples.py @@ -1,109 +1,107 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Local-only Evaluator plugin SDK smoke example. +"""Concise examples for the Evaluator plugin SDK surfaces. -The default entrypoint prints an exact-match spec and does not submit jobs or -call hosted models. Pass --run to execute the same offline metric against a -running local NeMo Platform. +These functions are intentionally not called at import time. Copy the one that +matches the feature being used and supply a configured NeMo Platform client. """ from __future__ import annotations -import argparse -import gzip -import json -import os -from collections.abc import Iterable from pathlib import Path -from tempfile import TemporaryDirectory from typing import Any -DEFAULT_BASE_URL = "http://localhost:8080" -DEFAULT_ROWS = ( - {"expected": "blue", "model_output": "blue"}, - {"expected": "Jupiter", "model_output": "Saturn"}, -) +def capital_france_metric() -> Any: + """Return an output-only metric suitable for stored agent-evaluation tasks.""" + from nemo_evaluator_sdk import StringCheckMetric -def write_jsonl_dataset(path: Path, rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> Path: - """Write rows as JSONL and return the written path.""" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") - return path - + return StringCheckMetric( + operation="equals", + left_template="{{sample.output_text | trim}}", + right_template="Paris", + ) -def load_jsonl_rows(path: Path, *, limit: int | None = None) -> list[dict[str, Any]]: - """Load plain JSONL or .gz JSONL rows.""" - opener = gzip.open if path.suffix == ".gz" else open - rows: list[dict[str, Any]] = [] - with opener(path, "rt", encoding="utf-8") as stream: - for line in stream: - if line.strip(): - rows.append(json.loads(line)) - if limit is not None and len(rows) >= limit: - break +def evaluate_standalone() -> Any: + """Evaluate one deterministic metric in process.""" + from nemo_evaluator_sdk import Evaluator, ExactMatchMetric - return rows + return Evaluator().run_sync( + metrics=ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", + ), + dataset=[ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, + ], + ) -def build_exact_match_spec(rows: Iterable[dict[str, Any]] = DEFAULT_ROWS) -> dict[str, Any]: - """Build a local exact-match spec that does not require model credentials.""" - from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric - from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager - from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric +def submit_and_collect(client: Any, output_dir: Path) -> tuple[Any, Path]: + """Submit one metric, wait for completion, and retrieve its artifacts.""" + from nemo_evaluator_sdk import ExactMatchMetric - metric = ExactMatchMetric( - reference="{{item.expected}}", - candidate="{{item.model_output}}", + job = client.evaluator.submit( + metric=ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", + ), + dataset=[{"expected": "Paris", "output": "Paris"}], ) - return { - "metrics": [bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json")], - "dataset": list(rows), - "params": {"parallelism": 2, "limit_samples": 2}, - } - - -def run_local_exact_match(dataset_path: Path) -> Any: - """Run the offline exact-match metric against a local platform.""" - from nemo_evaluator.sdk.types import RunConfig - from nemo_evaluator_sdk.enums import MetricType - from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric - from nemo_platform import NeMoPlatform - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", DEFAULT_BASE_URL), - workspace="default", + job.wait_until_done() + return job.get_result(), job.download_artifacts(output_dir) + + +def store_resources(client: Any) -> None: + """Store one metric, task, and taskset.""" + from nemo_evaluator.api.schemas import ( + MetricRef, + TaskInput, + TaskInputs, + TaskRef, + TasksetInput, ) - try: - evaluator = client.evaluator - metric = ExactMatchMetric( - type=MetricType.EXACT_MATCH, - reference="{{item.expected}}", - candidate="{{item.model_output}}", - ) - return evaluator.run(metric=metric, dataset=dataset_path, config=RunConfig(limit_samples=2)) - finally: - client.close() - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--run", action="store_true", help="Run local offline exact-match against NeMo Platform.") - args = parser.parse_args(argv) - - with TemporaryDirectory(prefix="nemo-evaluator-smoke-") as tmpdir: - dataset_path = write_jsonl_dataset(Path(tmpdir) / "exact-match.jsonl") - - if args.run: - result = run_local_exact_match(dataset_path) - result.print_summary() - return 0 + client.evaluator.metrics.create( + "answer-exact", + metric=capital_france_metric(), + ) + client.evaluator.tasks.create( + "capital-france", + task=TaskInput( + intent="Name the capital of France.", + inputs=TaskInputs(instruction="What is the capital of France?"), + metrics=[MetricRef("default/answer-exact")], + ), + ) + client.evaluator.tasksets.create( + "geography", + taskset=TasksetInput(tasks=[TaskRef("default/capital-france")]), + ) - print(json.dumps(build_exact_match_spec(load_jsonl_rows(dataset_path)), indent=2)) - return 0 +def build_agent_eval_spec(metric_bundle: Any) -> Any: + """Build a durable task evaluation with a runner target.""" + from nemo_evaluator.api.schemas import TaskInputs + from nemo_evaluator.jobs.agent_spec import ( + AgentEvalInputSpec, + AgentEvalTaskInput, + CodexRunnerTarget, + ) -if __name__ == "__main__": - raise SystemExit(main()) + return AgentEvalInputSpec( + tasks=[ + AgentEvalTaskInput( + id="capital-france", + intent="Name the capital of France.", + inputs=TaskInputs(instruction="What is the capital of France?"), + metrics=[metric_bundle], + ) + ], + target=CodexRunnerTarget(model=""), + max_concurrent_tasks=2, + benchmark={"name": "geography-smoke"}, + ) diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json deleted file mode 100644 index 6cf51562c0..0000000000 --- a/skills/nemo-evaluator-plugin/assets/specs/exact_match_benchmark.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "metrics": [ - { - "bundle_kind": "metric-bundle", - "bundle_format_version": "v1", - "metric_type": "exact-match", - "metadata": { - "description": null, - "labels": {} - }, - "outputs": [ - { - "name": "exact-match", - "description": null, - "value_json_schema": { - "description": "Continuous numeric metric value.", - "title": "ContinuousScore", - "type": "number" - } - } - ], - "secrets": {}, - "payload": { - "python_version": "3.11.15", - "cloudpickle_version": "3.1.2", - "pickle_protocol": 5, - "blob": "gAWVoQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwSe3tpdGVtLnJlZmVyZW5jZX19lIwJY2FuZGlkYXRllE51jBJfX3B5ZGFudGljX2V4dHJhX1-UTowXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QoaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", - "digest": "b6d94b6d5a4f304964652358cd55e8e1216664934a9712ac147d566b65ed3b5d", - "kind": "cloudpickle" - } - }, - { - "bundle_kind": "metric-bundle", - "bundle_format_version": "v1", - "metric_type": "string-check", - "metadata": { - "description": null, - "labels": {} - }, - "outputs": [ - { - "name": "string-check", - "description": null, - "value_json_schema": { - "description": "Continuous numeric metric value.", - "title": "ContinuousScore", - "type": "number" - } - } - ], - "secrets": {}, - "payload": { - "python_version": "3.11.15", - "cloudpickle_version": "3.1.2", - "pickle_protocol": 5, - "blob": "gAWV5gEAAAAAAACMJ25lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLnN0cmluZ19jaGVja5SMEVN0cmluZ0NoZWNrTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAxzdHJpbmctY2hlY2uUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJb3BlcmF0aW9ulIwIY29udGFpbnOUjA1sZWZ0X3RlbXBsYXRllIwWe3tzYW1wbGUub3V0cHV0X3RleHR9fZSMDnJpZ2h0X3RlbXBsYXRllIwYe3tpdGVtLnJlcXVpcmVkX3BocmFzZX19lHWMEl9fcHlkYW50aWNfZXh0cmFfX5ROjBdfX3B5ZGFudGljX2ZpZWxkc19zZXRfX5SPlChoHmggaAdoHJCMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", - "digest": "5c7c5b74de79b3d84fdd7db4f52393633dee4370036266d088518b94a92b081b", - "kind": "cloudpickle" - } - } - ], - "dataset": [ - { - "prompt": "Return exactly this word with no punctuation: Paris", - "reference": "Paris", - "required_phrase": "Paris" - }, - { - "note": "Intentional failure case: prompt asks for 'Oslo' but reference/required_phrase are 'London' so both metrics should report a miss.", - "prompt": "Return exactly this word with no punctuation: Oslo", - "reference": "London", - "required_phrase": "London" - } - ], - "params": { - "parallelism": 4, - "limit_samples": 2, - "ignore_request_failure": false, - "request_timeout": 60, - "max_retries": 3 - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim" - }, - "prompt_template": { - "messages": [ - { - "role": "user", - "content": "{{item.prompt}}" - } - ] - } -} diff --git a/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json index 4eaee1a54b..c697c588e9 100644 --- a/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +++ b/skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json @@ -1,46 +1,53 @@ { - "metrics": [ + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "exact-match", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ { - "bundle_kind": "metric-bundle", - "bundle_format_version": "v1", - "metric_type": "exact-match", - "metadata": { - "description": null, - "labels": {} - }, - "outputs": [ - { - "name": "exact-match", - "description": null, - "value_json_schema": { - "description": "Continuous numeric metric value.", - "title": "ContinuousScore", - "type": "number" - } - } - ], - "secrets": {}, - "payload": { - "python_version": "3.11.15", - "cloudpickle_version": "3.1.2", - "pickle_protocol": 5, - "blob": "gAWVuQEAAAAAAACMJm5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmV4YWN0X21hdGNolIwQRXhhY3RNYXRjaE1ldHJpY5STlCmBlH2UKIwIX19kaWN0X1-UfZQojAR0eXBllIwYbmVtb19ldmFsdWF0b3Jfc2RrLmVudW1zlIwKTWV0cmljVHlwZZSTlIwLZXhhY3QtbWF0Y2iUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwJcmVmZXJlbmNllIwRe3tpdGVtLmV4cGVjdGVkfX2UjAljYW5kaWRhdGWUjBV7e2l0ZW0ubW9kZWxfb3V0cHV0fX2UdYwSX19weWRhbnRpY19leHRyYV9flE6MF19fcHlkYW50aWNfZmllbGRzX3NldF9flI-UKGgeaBxoB5CMFF9fcHlkYW50aWNfcHJpdmF0ZV9flE51Yi4=", - "digest": "38050e2438a5eef8865ee2ef0bc2ccdaff6991b5d91bd9fb94a8155e759a26a5", - "kind": "cloudpickle" - } + "name": "exact-match", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } } - ], - "dataset": [ - { - "expected": "blue", - "model_output": "Blue" + ], + "secrets": {}, + "payload": { + "metric": { + "type": "exact-match", + "description": null, + "labels": {}, + "supported_job_types": [ + "online", + "offline" + ], + "reference": "{{item.expected}}", + "candidate": "{{item.output}}" }, - { - "expected": "Jupiter", - "model_output": "Saturn" - } - ], - "params": { - "parallelism": 2 + "digest": "bcd49ec7c31e962c06810501e5ea1c67f1667753b65a5a75db10686f6d992ed2", + "kind": "inline" + } + } + ], + "dataset": [ + { + "expected": "Paris", + "output": "Paris" + }, + { + "expected": "Paris", + "output": "London" } + ], + "params": { + "parallelism": 2 + } } diff --git a/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json b/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json new file mode 100644 index 0000000000..3a2d4e7a5f --- /dev/null +++ b/skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json @@ -0,0 +1,70 @@ +{ + "tasks": [ + { + "id": "capital-france", + "intent": "Name the capital of France.", + "inputs": { + "instruction": "What is the capital of France?" + }, + "reference": { + "expected": "Paris" + }, + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "exact-match", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ + { + "name": "exact-match", + "description": null, + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } + } + ], + "secrets": {}, + "payload": { + "metric": { + "type": "exact-match", + "description": null, + "labels": {}, + "supported_job_types": [ + "online", + "offline" + ], + "reference": "{{reference.expected}}", + "candidate": "{{sample.output_text}}" + }, + "digest": "e48a2f8e509d2ff3fe0b48749baec84577ee590050ff037449ddc5ddcee0045a", + "kind": "inline" + } + } + ] + } + ], + "target": { + "kind": "fabric", + "config": { + "metadata": { + "name": "readme-fabric-smoke" + }, + "harness": { + "adapter_id": "nvidia.fabric.codex" + } + }, + "model": "/", + "capture_trajectory": false + }, + "max_concurrent_tasks": 1, + "fail_fast": true, + "benchmark": { + "name": "readme-fabric-smoke" + } +} diff --git a/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json index 092072c428..7cde5dfb08 100644 --- a/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json +++ b/skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json @@ -1,63 +1,121 @@ { - "metrics": [ + "metrics": [ + { + "bundle_kind": "metric-bundle", + "bundle_format_version": "v1", + "metric_type": "llm-judge", + "metadata": { + "description": null, + "labels": {} + }, + "outputs": [ { - "bundle_kind": "metric-bundle", - "bundle_format_version": "v1", - "metric_type": "llm-judge", - "metadata": { - "description": null, - "labels": {} - }, - "outputs": [ - { - "name": "helpfulness", - "description": "How well does the response help the user?", - "value_json_schema": { - "description": "Continuous numeric metric value.", - "title": "ContinuousScore", - "type": "number" - } - } - ], - "secrets": { - "NVIDIA_API_KEY": "NVIDIA_API_KEY" - }, - "payload": { - "python_version": "3.11.15", - "cloudpickle_version": "3.1.2", - "pickle_protocol": 5, - "blob": "gAWVOAkAAAAAAACMJG5lbW9fZXZhbHVhdG9yX3Nkay5tZXRyaWNzLmxsbV9qdWRnZZSMDkxMTUp1ZGdlTWV0cmljlJOUKYGUfZQojAhfX2RpY3RfX5R9lCiMBHR5cGWUjBhuZW1vX2V2YWx1YXRvcl9zZGsuZW51bXOUjApNZXRyaWNUeXBllJOUjAlsbG0tanVkZ2WUhZRSlIwLZGVzY3JpcHRpb26UTowGbGFiZWxzlH2UjBNzdXBwb3J0ZWRfam9iX3R5cGVzlF2UKIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5jb21tb26UjBFTdXBwb3J0ZWRKb2JUeXBlc5STlIwGb25saW5llIWUUpRoFYwHb2ZmbGluZZSFlFKUZYwFbW9kZWyUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLm1vZGVsc5SMBU1vZGVslJOUKYGUfZQoaAV9lCiMA3VybJSMNGh0dHBzOi8vaW50ZWdyYXRlLmFwaS5udmlkaWEuY29tL3YxL2NoYXQvY29tcGxldGlvbnOUjARuYW1llIwhbnZpZGlhL25lbW90cm9uLTMtc3VwZXItMTIwYi1hMTJilIwPZGVmYXVsdF9oZWFkZXJzlE6MCGhvc3RfdXJslE6MDmFwaV9rZXlfc2VjcmV0lGgTjAlTZWNyZXRSZWaUk5QpgZR9lChoBX2UjARyb290lIwOTlZJRElBX0FQSV9LRVmUc4wXX19weWRhbnRpY19maWVsZHNfc2V0X1-Uj5QojARyb290lJB1YowGZm9ybWF0lGgIjAtNb2RlbEZvcm1hdJSTlIwDbmltlIWUUpR1jBJfX3B5ZGFudGljX2V4dHJhX1-UTmgxj5QoaCloI2glaDSQjBRfX3B5ZGFudGljX3ByaXZhdGVfX5ROdWKMBnNjb3Jlc5RdlIwgbmVtb19ldmFsdWF0b3Jfc2RrLnZhbHVlcy5zY29yZXOUjApSYW5nZVNjb3JllJOUKYGUfZQoaAV9lChoJYwLaGVscGZ1bG5lc3OUaA6MKUhvdyB3ZWxsIGRvZXMgdGhlIHJlc3BvbnNlIGhlbHAgdGhlIHVzZXI_lIwGcGFyc2VylGg_jA9KU09OU2NvcmVQYXJzZXKUk5QpgZR9lChoBX2UKGgHjARqc29ulIwJanNvbl9wYXRolGhFdWg6Tmgxj5QoaE6QaDxOdWKMB21pbmltdW2USwCMB21heGltdW2USwR1aDpOaDGPlChoR2hRaA5oJWhQkGg8TnViYYwPcHJvbXB0X3RlbXBsYXRllH2UjAhtZXNzYWdlc5RdlCh9lCiMBHJvbGWUjAZzeXN0ZW2UjAdjb250ZW50lIyGWW91IGFyZSBhbiBldmFsdWF0b3IuIFJhdGUgdGhlIHJlc3BvbnNlJ3MgaGVscGZ1bG5lc3MgZnJvbSAwLTQuIFJldHVybiBvbmx5IGEgSlNPTiBvYmplY3Qgd2l0aCB0aGlzIHNoYXBlOiB7ImhlbHBmdWxuZXNzIjogPGludGVnZXI-fS6UdX2UKGhYjAR1c2VylGhajHNVc2VyIHByb21wdDoge3tpdGVtLmlucHV0fX0KCkFzc2lzdGFudCByZXNwb25zZToge3tzYW1wbGUub3V0cHV0X3RleHQgfCBkZWZhdWx0KGl0ZW0ub3V0cHV0KX19CgpSYXRlIHRoaXMgcmVzcG9uc2UulHVlc4wPb3B0aW9uYWxfZmllbGRzlF2UjBFzdHJ1Y3R1cmVkX291dHB1dJR9lIwGc2NoZW1hlH2UKGgHjAZvYmplY3SUjApwcm9wZXJ0aWVzlH2UaEV9lChoB4wHaW50ZWdlcpRoUEsAaFFLBHVzjAhyZXF1aXJlZJRdlGhFYXVzjAlpbmZlcmVuY2WUjCBuZW1vX2V2YWx1YXRvcl9zZGsudmFsdWVzLnBhcmFtc5SMD0luZmVyZW5jZVBhcmFtc5STlCmBlH2UKGgFfZQojAt0ZW1wZXJhdHVyZZRHAAAAAAAAAACMCm1heF90b2tlbnOUTQCAjBVtYXhfY29tcGxldGlvbl90b2tlbnOUTowFdG9wX3CUTowEc3RvcJROdWg6fZRoMY-UKGhzaHSQaDxOdWKMDXN5c3RlbV9wcm9tcHSUTowJcmVhc29uaW5nlE6MFmlnbm9yZV9yZXF1ZXN0X2ZhaWx1cmWUiYwIam9iX3R5cGWUaBh1aDpOaDGPlChoX2gcaFNobGg9aGFofJBoPH2UKIwRX3ByZXByb2Nlc3NfaG9va3OUXZQojBxuZW1vX2V2YWx1YXRvcl9zZGsuaW5mZXJlbmNllIwVQWRkSW5mZXJlbmNlUGFyYW1ldGVylJOUKYGUfZSMBnBhcmFtc5R9lChoc0cAAAAAAAAAAGh0TQCAdXNijCRuZW1vX2V2YWx1YXRvcl9zZGsuc3RydWN0dXJlZF9vdXRwdXSUjBlJbmZlcmVuY2VTdHJ1Y3R1cmVkT3V0cHV0lJOUKYGUfZQojAxfanNvbl9zY2hlbWGUfZQoaAdoZWhmaGdoamhrdYwHX3N0cmljdJSJjARtb2RllGiJjBRTdHJ1Y3R1cmVkT3V0cHV0TW9kZZSTlIwRbnZleHRfZ3VpZGVkX2pzb26UhZRSlIwPaW5mZXJlbmNlX3BhcmFtlH2UjApleHRyYV9ib2R5lH2UjAVudmV4dJR9lIwLZ3VpZGVkX2pzb26UaI9zc3N1YmiCjAdMb2dIb29rlJOUKYGUfZSMBmxvZ2dlcpSMB2xvZ2dpbmeUjAlnZXRMb2dnZXKUk5RogoWUUpRzYmWMEl9wb3N0cHJvY2Vzc19ob29rc5RdlGigYYwaX3VzZV9tYXhfY29tcGxldGlvbl90b2tlbnOUiYwIX2FwaV9rZXmUTowHX2NsaWVudJROjA1faW5mZXJlbmNlX2ZulE6MCF9wYXJzZXJzlH2UaEVoP4wPU2NvcmVQYXJzZXJKU09OlJOUKYGUfZQojAVzY29yZZRoQmhOaEVoYWhijAtqc29uX3NjaGVtYZRoZHVic4wMX3Njb3JlX2R1bXBzlH2UaEV9lChoJWhFaA5oRmhQSwBoUUsEdXOMG19wcm9tcHRfdGVtcGxhdGVfaXNfZGVmYXVsdJSJdXViLg==", - "digest": "dfd6a04359b75b41cba2817bc1496244425e8290102b36c20bd04e7f62b31b8e", - "kind": "cloudpickle" - } + "name": "helpfulness", + "description": "How well the response helps the user.", + "value_json_schema": { + "description": "Continuous numeric metric value.", + "title": "ContinuousScore", + "type": "number" + } } - ], - "dataset": [ - { - "input": "What is the capital of France?" - }, - { - "input": "How do I make scrambled eggs?" - } - ], - "params": { - "parallelism": 2, - "limit_samples": 2, - "request_timeout": 120, - "max_retries": 3 - }, - "target": { - "url": "https://integrate.api.nvidia.com/v1/chat/completions", - "name": "nvidia/nemotron-3-super-120b-a12b", - "api_key_secret": "NVIDIA_API_KEY", - "format": "nim" - }, - "prompt_template": { - "messages": [ + ], + "secrets": { + "NVIDIA_API_KEY": "NVIDIA_API_KEY" + }, + "payload": { + "metric": { + "type": "llm-judge", + "description": null, + "labels": {}, + "supported_job_types": [ + "online", + "offline" + ], + "model": { + "url": "https://integrate.api.nvidia.com/v1/chat/completions", + "name": "nvidia/nemotron-3-super-120b-a12b", + "host_url": null, + "api_key_secret": "NVIDIA_API_KEY", + "format": "nim" + }, + "scores": [ { + "name": "helpfulness", + "description": "How well the response helps the user.", + "parser": { + "type": "json", + "json_path": "helpfulness" + }, + "minimum": 0, + "maximum": 4 + } + ], + "prompt_template": { + "messages": [ + { + "role": "system", + "content": "Rate helpfulness from 0-4. Return JSON only: {\"helpfulness\": }." + }, + { "role": "user", - "content": "{{item.input}}" + "content": "Request: {{item.input}}\nResponse: {{sample.output_text}}" + } + ] + }, + "optional_fields": [], + "structured_output": { + "schema": { + "type": "object", + "properties": { + "helpfulness": { + "type": "integer", + "minimum": 0, + "maximum": 4 + } + }, + "required": [ + "helpfulness" + ] } - ] + }, + "inference": null, + "system_prompt": null, + "reasoning": null, + "ignore_request_failure": false, + "job_type": "online" + }, + "digest": "bc8712d3e93806036bb07b5f572fea60049149a6785a720c85303c17f04ea078", + "kind": "inline" + } + } + ], + "dataset": [ + { + "input": "What is the capital of France?" + }, + { + "input": "How do I make scrambled eggs?" } + ], + "params": { + "parallelism": 2, + "limit_samples": 2, + "request_timeout": 120, + "max_retries": 3 + }, + "target": { + "url": "https://integrate.api.nvidia.com/v1/chat/completions", + "name": "nvidia/nemotron-3-super-120b-a12b", + "host_url": null, + "api_key_secret": "NVIDIA_API_KEY", + "format": "nim" + }, + "prompt_template": { + "messages": [ + { + "role": "user", + "content": "{{item.input}}" + } + ] + } } diff --git a/skills/nemo-evaluator-plugin/references/agent-evaluation.md b/skills/nemo-evaluator-plugin/references/agent-evaluation.md new file mode 100644 index 0000000000..d9dd143909 --- /dev/null +++ b/skills/nemo-evaluator-plugin/references/agent-evaluation.md @@ -0,0 +1,226 @@ +# Agent Evaluation + +Read this file for agentic task-driven evaluation, direct SDK runners, platform +`agent-evaluate` jobs, tasksets, precomputed trials, or Harbor and custom runners. + +## Choose standalone SDK or platform job + +Use `AgentEvaluator` lightweight in-process evaluation that does not require a running nemo-platform: + +```python +from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator + +result = await AgentEvaluator().run(tasks=tasks, target=target) +print(result.trials) +print(result.summary) +``` + +The standalone target union is: + +- `Model` +- `GenericAgent` +- Any object implementing `nemo_evaluator_sdk.agent_eval.trials.AgentTaskRunner` protocol + +For a minimal direct runner: + +```python +from nemo_evaluator_sdk.agent_eval.runtimes.callable_runtime import ( + CallableAgentTaskRunner, +) + +async def answer(task): + return task.inputs["instruction"] + +runner = CallableAgentTaskRunner(answer) +result = await AgentEvaluator().run(tasks=tasks, target=runner) +``` + +Submit the plugin job when platform execution is required: + +```bash +nemo evaluator agent-evaluate explain +nemo evaluator agent-evaluate submit --spec-file agent-eval.json +``` + +## Build the job input + +`AgentEvalInputSpec.tasks` accepts an inline task list or a stored `TasksetRef`. +Provide exactly one trial source: + +- `target` to generate trials. +- `trials` to rescore precomputed trials. + +```python +from nemo_evaluator.jobs.agent_spec import ( + AgentEvalInputSpec, + AgentEvalTaskInput, + CodexRunnerTarget, +) + +spec = AgentEvalInputSpec( + tasks=[ + AgentEvalTaskInput( + id="capital-france", + intent="Name the capital of France.", + inputs={"instruction": "What is the capital of France?"}, + metrics=[metric_bundle], + ) + ], + target=CodexRunnerTarget(model=""), + max_concurrent_tasks=2, + fail_fast=False, + benchmark={"name": "geography-smoke"}, +) +``` + +Use `TasksetRef("default/geography")` with `submit` for persisted tasks. Stored +tasks do not include grader-only `reference`; use inline tasks when metrics +require held-out per-task data. + +## Choose a platform target + +| Target | Use when | +| --- | --- | +| `ModelTarget` | Generate trials through an OpenAI-compatible model endpoint | +| `AgentTarget` | Generate trials through a generic HTTP or NeMo Agent Toolkit agent | +| `CodexRunnerTarget` | Drive the Codex CLI runner | +| `FabricRunnerTarget` | Run a configured NeMo [Fabric](https://github.com/nvidia/nemo-fabric) runner | +| `HarborRunnerTarget` | Run a Harbor task suite in Docker | + +`ModelTarget` owns its `prompt_template` and online model params. +`AgentTarget` owns its agent request configuration. Runner targets are resolved +to an `AgentTaskRunner` inside the job runtime. + +For [Fabric](https://github.com/nvidia/nemo-fabric), pass one complete `agent.yaml` as a JSON-shaped `config`; the +`harness.adapter_id` selects the harness: + +```python +from nemo_evaluator.jobs.agent_spec import FabricRunnerTarget + +target = FabricRunnerTarget( + config={ + "metadata": {"name": "regression-suite"}, + "harness": {"adapter_id": "nvidia.fabric.codex"}, + }, + model="/", +) +``` + +Do not use profile overlays. Fold the complete configuration into `config`. + +`max_concurrent_tasks` limits tasks evaluated concurrently. Target-specific +settings such as inference parallelism or Harbor +`n_concurrent_trials` control concurrency inside trial generation. + +## Use precomputed trials + +Pass `trials=[...]` and omit `target` to rescore stored outputs and/or trajectories +without invoking the original model, agent, or runner. Keep stable `task_id` +values so trials match task definitions. + +Individual trials are stored in the run bundle, not as queryable result entities. +Retrieve the run index, download its bundle, and hydrate `trials.jsonl`: + +```python +from nemo_evaluator_sdk.agent_eval.persistence import read_trials + +stored = client.evaluator.agent_eval_results.retrieve("") +client.files.download(remote_path=stored.bundle_ref, local_path="previous-run") +trials = read_trials("previous-run") +``` + +CLI equivalent for downloading the bundle: + +```bash +nemo jobs results download agent-eval-results \ + --job --output-file agent-eval-results.tar.gz +mkdir -p previous-run +tar -xzf agent-eval-results.tar.gz -C previous-run --strip-components=1 +``` + +Pass the hydrated `trials` with the same task definitions and omit `target`. + +## Read results + +A standalone run returns an `AgentEvalResult`: + +- `result.summary` contains aggregate values per metric output plus coverage + counts for scored, failed, and missing-output trials. +- `result.scores` contains one entry per task, trial, and metric, including + metric outputs, status, and diagnostics. +- `result.trials` contains each agent output, its evidence, and its + `completed`, `partial`, or `failed` status. +- `result.run_id` identifies the run; `result.benchmark` contains its grouping + metadata. + +When standalone `AgentEvalRunConfig.output_dir` is set, the same information is +written as a run bundle: + +| File | Contents | +| --- | --- | +| `summary.json` | Aggregate mean, minimum, maximum, standard deviation, counts, and coverage | +| `scores.jsonl` | Per-task, trial, and metric outputs, status, and diagnostics | +| `trials.jsonl` | Trial outputs, evidence, metadata, and status | +| `tasks.jsonl` | Tasks included in the run | +| `run.json` | Run ID and artifact manifest | +| `benchmark.json` | Benchmark-grouping metadata | +| `report.html` | Browsable dashboard when dashboard generation is enabled | + +Use the in-memory result for programmatic follow-up and the bundle for +inspection, sharing, or rescoring. Platform jobs persist the bundle and create +a queryable record under `client.evaluator.agent_eval_results`. + +Inspect failed and partial trials and score diagnostics before interpreting +aggregate values; a high mean with low coverage can hide missing or failed +work. + +## Configure Harbor as a task runner + +Harbor requires its Python package, Docker access, and a Harbor dataset. Task +discovery records the source dataset in each task's +`harbor_dataset_path` metadata; the durable runtime recovers the dataset from +that metadata. + +**Standalone SDK:** + +```python +from pathlib import Path + +from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import ( + HarborAgentTaskRunner, + HarborRuntimeConfig, + discover_harbor_tasks, +) + +tasks = discover_harbor_tasks("path/to/harbor-suite") +runner = HarborAgentTaskRunner( + config=HarborRuntimeConfig( + jobs_dir=Path("harbor-jobs"), + agent_name="oracle", + n_attempts=1, + n_concurrent_trials=2, + ) +) +result = await AgentEvaluator().run(tasks=tasks, target=runner) +``` + +**Platform SDK:** + +```python +from nemo_evaluator.jobs.agent_spec import HarborRunnerTarget + +target = HarborRunnerTarget( + agent_name="oracle", + n_attempts=1, + n_concurrent_trials=2, + max_retries=0, + artifacts=["/workspace/output"], + trace_dir="/app/traces", + reward_key="reward", +) +``` + +Use `agent_import_path` for a custom Harbor agent and `agent_model_name` when +the agent requires a model. The module must be importable in the execution +environment. Durable execution additionally requires an execution image and +runtime that provide Harbor and Docker access. diff --git a/skills/nemo-evaluator-plugin/references/api-auth.md b/skills/nemo-evaluator-plugin/references/api-auth.md index 4c69361724..e49701938b 100644 --- a/skills/nemo-evaluator-plugin/references/api-auth.md +++ b/skills/nemo-evaluator-plugin/references/api-auth.md @@ -1,15 +1,68 @@ # Evaluator API Auth -Use the correct `model.api_key_secret` (if `model` is used) for the evaluator execution mode: +Read this file before configuring a model, agent, remote metric, LLM judge, or +durable platform job. -- Local `nemo evaluator evaluate run`: `api_key_secret` is the name of an environment variable available to the local process, such as `NVIDIA_API_KEY`. -- Remote `nemo evaluator evaluate submit`: `api_key_secret` is the name of a NeMo platform secret in the target workspace, such as `nvidia-api-key`. +## Match the secret reference to the execution mode -The remote job runtime cannot read local environment variables. In remote mode, if a model sets `api_key_secret`, create or verify the platform secret before submitting the job: +`api_key_secret` is a reference, never the credential value. It resolves to a different value depending on execution mode: standalone and plugin submission. + +| Execution | `api_key_secret` resolves to | +| --- | --- | +| Standalone SDK | Environment-variable name in the calling process, such as `NVIDIA_API_KEY` | +| Plugin `submit` | NeMo Platform secret name in the target workspace, such as `nvidia-api-key` | + +A remote job cannot read the submitting shell's environment variables. Before +submitting, verify the `api_key_secret` is in the list of secrets: + +```bash +nemo secrets list +``` + +Create it through the supported secrets CLI for the installed NeMo Platform +version. Do not put the key directly in a spec, command line, log, or committed +file. ```bash printf '%s' "$NVIDIA_API_KEY" | nemo secrets create nvidia-api-key --from-file - nemo secrets list ``` -If you copy a local LLM-judge spec that uses `"api_key_secret": "NVIDIA_API_KEY"` for remote submission, change that value to the platform secret name, for example `"nvidia-api-key"`. +## Adapt the local-first spec for platform submission + +The checked `llm_as_judge.json` uses the local environment variable +`NVIDIA_API_KEY`. Create a platform copy that points both the generation target +and the judge's environment binding at the workspace secret: + +```bash +jq --arg platform_secret "nvidia-api-key" ' + .target.api_key_secret = $platform_secret + | .metrics[0].secrets.NVIDIA_API_KEY = $platform_secret +' skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json \ + > llm_as_judge.platform.json +``` + +The bundle key `NVIDIA_API_KEY` remains the environment-variable name expected +by the judge; its value becomes the platform secret name. Do not edit +`metrics[*].payload` or its digest will no longer describe the inline metric. + +## Diagnose remote 409 responses + +Do not assume HTTP 409 means a duplicate job. Inspect the response body. The +Jobs service can return 409 when a referenced platform secret does not exist +or is inaccessible, for example: + +```text +Unable to create job because one or more referenced secrets were not found or +are not accessible. +``` + +The response intentionally may not identify the secret. Verify every referenced +workspace and secret name, then retry the submission. + +## Follow security best practices + +- Print secret names only, never values. +- Redact authorization headers and provider responses that echo credentials. +- Use placeholders such as `` in shared examples. +- Do not copy `.env` files into job artifacts. diff --git a/skills/nemo-evaluator-plugin/references/evaluation-shapes.md b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md new file mode 100644 index 0000000000..494fa45b97 --- /dev/null +++ b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md @@ -0,0 +1,42 @@ +# Dataset-Driven vs Task-Driven Evaluation + +Choose the evaluation shape from what produces the scored output. Metrics are +shared scorers; the input and evidence differ. + +| Question | Dataset-driven | Task-driven | +| --- | --- | --- | +| What is the input? | Fixed dataset rows | Tasks with intent, inputs, and metrics | +| What is scored? | One output per row | One or more trials per task | +| Which metrics apply? | The same metric set applies to every row | Each task can define its own metrics | +| What evidence is available? | Row fields, row scores, and aggregates | Final output, trajectory, tool calls, other trial evidence, per-task rewards, and summary | +| Platform job | `evaluate submit` | `agent-evaluate submit` | + +## Dataset-driven evaluation + +Use dataset-driven evaluation for a fixed set of examples where the same +scoring rules apply to every row. Typical uses include model quality checks, +RAG regression tests, and labeled-set benchmarks. + +Each row contains the fields consumed by the metric, for example: + +```python +{"question": "Capital of France?", "expected": "Paris", "output": "Paris"} +``` + +Read [SDK Execution](execution.md) for datasets, targets, field mappings, +submission, and results. +Read [Metric Selection](metric-selection.md) to choose +the scorer. + +## Task-driven evaluation + +Use task-driven evaluation when the system performs work and the process can +matter as much as the final answer. A model, agent, or runner produces a trial +containing the final output and available execution evidence. Tasks can carry +different metrics, so one taskset can grade heterogeneous work. + +Choose this shape for agent behavior, tool use, multi-step work, runner-based +benchmarks, or rescoring precomputed trials. + +Read [Agent Evaluation](agent-evaluation.md) for tasks, trials, tasksets, +targets, runners, concurrency, and results. diff --git a/skills/nemo-evaluator-plugin/references/execution.md b/skills/nemo-evaluator-plugin/references/execution.md new file mode 100644 index 0000000000..f4842417e9 --- /dev/null +++ b/skills/nemo-evaluator-plugin/references/execution.md @@ -0,0 +1,253 @@ +# SDK Execution + +Read this file before choosing a dataset representation, execution mode, +target, configuration, field mapping, or job-result operation. + +CLI snippets use the installed `nemo` command. In a repository checkout, +prefix them with `uv run`. + +## Validate standalone, then submit to the platform + +### Standalone SDK +Use the standalone SDK for the fastest in-process metric loop: + +```python +from nemo_evaluator_sdk import Evaluator, ExactMatchMetric + +result = Evaluator().run_sync( + metrics=ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", + ), + dataset=[ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, + ], +) +print(result.row_scores) +print(result.aggregate_scores) +``` + +**Platform CLI** +Platform CLI equivalent for the same checked metric and rows: + +```bash +nemo evaluator evaluate submit \ + --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json +``` + +**Platform Python SDK** +Use `client.evaluator.submit` for execution through the installed nemo-evaluator-plugin: + +```python +from nemo_evaluator.sdk import RunConfig +from nemo_evaluator_sdk import ExactMatchMetric +from nemo_platform import NeMoPlatform + +client = NeMoPlatform(base_url="http://localhost:8080", workspace="default") +job = client.evaluator.submit( + metric=ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", + ), + dataset=[ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, + ], + config=RunConfig(parallelism=2), +) +job.wait_until_done() +result = job.get_result() +``` + +### Use a Fileset for the dataset +The job will run in the container environment. +The plugin submission dataset accepts inline rows, a `str` or `Path`, and +`FilesetRef`: + +**Platform SDK** +```python +from nemo_evaluator.sdk import FilesetRef + +dataset = FilesetRef("default/eval-data") +``` + +**Platform CLI** +CLI equivalent, using a stored metric and fileset: +```bash +nemo evaluator evaluate submit \ + --spec '{"metrics":["default/exact-answer"],"dataset":"default/eval-data"}' +``` + +## Configure online generation + +Use `RunConfigOnlineModel` with `Model`, or `RunConfigOnline` with `Agent` when the evaluator should generate output before scoring. +Provide a prompt template when the evaluator should generate output before +scoring. + +**Platform SDK** +```python +from nemo_evaluator_sdk import ExactMatchMetric, Model, RunConfigOnlineModel + +job = client.evaluator.submit( + metric=ExactMatchMetric(reference="{{item.expected}}"), + dataset=[{"question": "Capital of France?", "expected": "Paris"}], + target=Model( + url="https://provider.example/v1/chat/completions", + name="", + format="openai", + api_key_secret="", + ), + prompt_template={ + "messages": [{"role": "user", "content": "{{item.question}}"}], + }, + config=RunConfigOnlineModel(parallelism=2), +) +job.wait_until_done() +result = job.get_result() +``` + +**Platform CLI** + +The checked LLM-judge spec is local-first and reads `NVIDIA_API_KEY`. Follow +[Adapt the local-first spec for platform submission](api-auth.md#adapt-the-local-first-spec-for-platform-submission) +to create `llm_as_judge.platform.json`, then submit that copy: + +```bash +nemo evaluator evaluate submit \ + --spec-file llm_as_judge.platform.json +``` + +Platform submission requires provider access and the referenced platform workspace +secret. + +## Map noncanonical fields + +Use `FieldMapping` when the metric expects canonical evaluator fields but the +dataset uses different column names: + +**Platform SDK** +```python +from nemo_evaluator_sdk import FieldMapping + +mapping = FieldMapping( + output="assistant_answer", + reference="gold_answer", +) +``` +Pass it as `field_mapping=mapping` when submitting the job. + +**Platform CLI** +```bash +nemo evaluator evaluate submit --spec \ + '{ + "metrics": ["default/exact-answer"], + "dataset": [{"gold_answer": "Paris", "assistant_answer": "Paris"}], + "field_mapping": { + "output": "assistant_answer", + "reference": "gold_answer" + } + }' +``` + +Use field_mapping when a metric or online prompt uses canonical evaluator fields but the dataset uses different column names. One job-level mapping applies to every metric and the generation prompt. + +## Getting job results + +**Platform SDK** +```python +job = client.evaluator.submit( + metric=metric, + dataset=dataset, + config=config, + target=target, + prompt_template=prompt_template, +) + +job.wait_until_done() +result = job.get_result() +artifacts = job.download_artifacts("evaluation-artifacts") +``` +`EvaluatorJobResource` also exposes methods for job lifecycle management: +- `name` and `job` +- `get_job_status()` +- `check_if_complete(raise_if_not_complete=False)` +- `get_result(aggregate_fields=...)` +- `as_async()` + +**Platform CLI** +Poll until the job is completed before downloading results: + +```bash +nemo evaluator evaluate submit --spec-file evaluation.json +nemo jobs get-status +nemo jobs results list +nemo jobs results download aggregate-scores \ + --job --output-file aggregate-scores.json +nemo jobs results download row-scores \ + --job --output-file row-scores.jsonl +``` + +The CLI `submit` command returns the created job record immediately. It does +not wait or expose follow-up result/download commands under the `evaluate` +group. Use the SDK submission handle when the workflow needs those lifecycle +operations. + +### Notes +- Always wait for terminal completion. A metric can report 100 percent progress +before the platform finishes publishing result artifacts. +- `submit` accepts a concrete `Model` or `ModelRef`; the platform resolves model +references in the target workspace. + + +## Multiple metrics + +The high-level plugin helper takes one runtime `metric`. To combine metrics, +build an `EvaluateInputSpec` and submit it with the CLI. Stored metric references +are resolved by the platform submission path: + +```json +{ + "metrics": [ + "default/accuracy", + "default/style" + ], + "dataset": "default/eval-data", + "params": {"parallelism": 4} +} +``` + +Save the spec as `multi-metric.json`, then submit it: + +```bash +nemo evaluator evaluate submit --spec-file multi-metric.json +``` + +Inspect the authoritative wire schema before authoring a spec: + +```bash +nemo evaluator evaluate explain +``` + +## Package metrics safely + +Built-in metrics default to declarative inline bundles. For a custom Python +metric submitted to a service, opt in explicitly: + +```python +from nemo_evaluator.shared.metric_bundles.hybrid import HybridMetricBundlePackager + +job = client.evaluator.submit( + metric=custom_metric, + dataset=rows, + metric_bundle_packager=HybridMetricBundlePackager(), +) +``` + +The CLI cannot package a Python metric object or select +`metric_bundle_packager`. After Python serializes the bundled metric into a +complete spec, submit that spec with: + +```bash +nemo evaluator evaluate submit --spec-file custom-metric.json +``` diff --git a/skills/nemo-evaluator-plugin/references/llm-judge.md b/skills/nemo-evaluator-plugin/references/llm-judge.md index 2052924ca1..b5d0ce6fe3 100644 --- a/skills/nemo-evaluator-plugin/references/llm-judge.md +++ b/skills/nemo-evaluator-plugin/references/llm-judge.md @@ -1,32 +1,78 @@ -# LLM Judge Notes +# LLM Judge -Use `nemo evaluator evaluate explain` to inspect the current Evaluator plugin spec schema before creating an LLM-judge run. +Read this file when deterministic metrics cannot express the rubric and an LLM +must score existing or generated responses. -When configuring an LLM judge, verify: +## Configure the judge -1. The judge model authentication reference matches the execution mode. See [Evaluator API Auth](api-auth.md). +Keep the judge model, score contract, parser, and prompt explicit: -2. The judge model name is the API model ID expected by the endpoint, not an entity display name. +```python +from nemo_evaluator_sdk import JSONScoreParser, LLMJudgeMetric, Model, RangeScore -3. The metric prompt and parser match the output you expect from the judge model. +judge = LLMJudgeMetric( + model=Model( + url="https://provider.example/v1/chat/completions", + name="", + format="openai", + api_key_secret="", + ), + scores=[ + RangeScore( + name="helpfulness", + description="How well the response addresses the request.", + minimum=0, + maximum=4, + parser=JSONScoreParser(json_path="helpfulness"), + ) + ], + prompt_template={ + "messages": [ + { + "role": "system", + "content": 'Return JSON only: {"helpfulness": }.', + }, + { + "role": "user", + "content": "Request: {{item.input}}\nResponse: {{item.output}}", + }, + ] + }, +) +``` -For local iteration, keep the metric and dataset in a spec file and run: +Use lowercase letters, numbers, and underscores in score names. Ensure the +judge response exactly matches the parser: the example parser expects a JSON +field named `helpfulness`. -```bash -nemo evaluator evaluate run --spec-file evaluation-spec.json -``` +## Validate before scaling -The checked-in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json` is a local-run example. It expects `NVIDIA_API_KEY` to be set in the local shell. +1. Use one response that should score high and one that should score low. +2. Confirm the model ID is accepted by the configured endpoint. +3. Inspect raw judge output and row-level parser errors. +4. Confirm the score range and aggregate match the rubric. +5. Only then increase dataset size or submit a durable job. -For durable execution, submit the same spec: +Use: ```bash -nemo evaluator evaluate submit \ - --spec-file evaluation-spec.json \ - --workspace default \ - --profile default +nemo evaluator metric-types llm-judge +nemo evaluator evaluate explain ``` -Before submitting an LLM-judge spec via `submit`, replace local environment-variable names with platform secret names, such as `nvidia-api-key`. +Prefer `--spec-file` over shell-escaped inline JSON. The checked +`assets/specs/llm_as_judge.json` demonstrates the minimum online generation +target plus judge configuration. + +## Keep judge and generation roles separate + +For offline judge-quality evaluation, put existing responses in dataset rows +and omit the generation target. For online generation-quality evaluation, pass +a separate `Model` or `Agent` target plus `prompt_template`; the judge metric +then scores the generated sample. + +Do not treat labels for old responses as labels for newly generated responses +unless the benchmark protocol explicitly defines that mapping. -Prefer `--spec-file` over inline `--spec` for LLM-judge metrics because prompts and score definitions quickly become hard to audit as shell-escaped JSON. +See [Evaluator API Auth](api-auth.md) before switching the same configuration +between standalone and platform execution. diff --git a/skills/nemo-evaluator-plugin/references/metric-selection.md b/skills/nemo-evaluator-plugin/references/metric-selection.md new file mode 100644 index 0000000000..9e521e8311 --- /dev/null +++ b/skills/nemo-evaluator-plugin/references/metric-selection.md @@ -0,0 +1,82 @@ +# Metric Selection + +Read this file when converting a rubric into evaluator metrics. + +## Prefer the simplest metric +Python import path of a metric: `nemo_evaluator_sdk.metrics.` + +| Goal | Prefer | +| --- | --- | +| Exact label, enum, or regression | `ExactMatchMetric` | +| Contains, equals, or starts/ends with | `StringCheckMetric` | +| Numeric value or threshold | `NumberCheckMetric` | +| Text overlap | `F1Metric`, `BLEUMetric`, or `ROUGEMetric` | +| Semantic quality or a written rubric | `LLMJudgeMetric` | +| Retrieval smoke test | A deterministic context assertion or `LLMJudgeMetric` | +| Tool-call correctness | `ToolCallingMetric` or `ToolCallAccuracyMetric` | +| Existing scoring service | `RemoteMetric` or `NemoAgentToolkitRemoteMetric` | +| Agent answer or goal completion | A task-specific custom metric that implements `nemo_evaluator_sdk.metrics.protocol.Metric` or `LLMJudgeMetric` | + +Use deterministic metrics before an LLM judge. Use an LLM only when the +criterion requires semantic judgment. + +RAGAS metrics remain available for specialized RAG evaluation, but do not use +them as skill smoke examples: even a one-row live judge call can take minutes. +Use the evaluator RAG metric documentation when the user explicitly requests a +RAGAS score, validate its input schema first, and label the provider and +timeout prerequisites. + +### Explore the metrics provided by the SDK + +List current metric names and inspect one schema: + +```bash +nemo evaluator metric-types +nemo evaluator metric-types exact-match +``` + +## Validate the mapping + +Create one row that must pass and one that must fail: + +```python +from nemo_evaluator_sdk import ExactMatchMetric + +metric = ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", +) +rows = [ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, +] +``` + +Check: + +- Dataset keys match every Jinja template. +- Normalization is intentional; do not hide case or whitespace differences + unless the rubric says they are irrelevant. +- Judge prompts specify the rubric and parser-compatible output. +- RAG and tool metrics receive their required canonical fields or a + `FieldMapping`. + +## Use multiple metrics only for distinct dimensions + +SDK — pass a metric sequence in one call: + +```python +from nemo_evaluator_sdk import Evaluator +result = Evaluator().run_sync(metrics=[accuracy, style], dataset=rows) +``` + +Platform job — put multiple stored metrics on the job spec: + +```bash +nemo evaluator evaluate submit --spec \ + '{"metrics":["default/accuracy","default/style"],"dataset":"default/eval-data"}' +``` + +Each `metrics` entry may be an inline metric bundle, a stored `MetricRef`, or +a mix of both. The high-level `client.evaluator.submit` helper still accepts +only one runtime metric per call. diff --git a/skills/nemo-evaluator-plugin/references/resources.md b/skills/nemo-evaluator-plugin/references/resources.md new file mode 100644 index 0000000000..2505c43e2e --- /dev/null +++ b/skills/nemo-evaluator-plugin/references/resources.md @@ -0,0 +1,132 @@ +# Stored Resources + +Read this file when definitions or results must be reusable and queryable +through `client.evaluator`. + +## Resource map + +| Resource | Create | Retrieve | List | Delete | Update | +| --- | --- | --- | --- | --- | --- | +| `metrics` | yes | yes | yes | yes | no | +| `tasks` | yes | yes | yes | yes | no | +| `tasksets` | yes | yes | yes | yes | no | +| `eval_results` | no | yes | yes | yes | no | +| `agent_eval_results` | no | yes | yes | yes | no | + +Metrics, tasks, and tasksets are immutable. Delete and recreate them, or use a +new versioned name. + +## Store a metric, task, and taskset + +```python +from nemo_evaluator.api.schemas import ( + MetricRef, + TaskInput, + TaskInputs, + TaskRef, + TasksetInput, +) +from nemo_evaluator_sdk import StringCheckMetric + +client.evaluator.metrics.create( + "answer-exact", + metric=StringCheckMetric( + operation="equals", + left_template="{{sample.output_text | trim}}", + right_template="Paris", + ), +) + +client.evaluator.tasks.create( + "capital-france", + task=TaskInput( + intent="Name the capital of France.", + inputs=TaskInputs(instruction="What is the capital of France?"), + metrics=[MetricRef("default/answer-exact")], + ), +) + +client.evaluator.tasksets.create( + "geography", + taskset=TasksetInput( + description="Geography smoke tasks.", + tasks=[TaskRef("default/capital-france")], + ), +) +``` + +For a task that needs held-out ground truth invisible to the agent, keep the reference on an +inline `AgentEvalTaskInput` and use a metric that reads it: + +```python +from nemo_evaluator.api.schemas import MetricRef, TaskInputs +from nemo_evaluator.jobs.agent_spec import AgentEvalTaskInput +from nemo_evaluator_sdk import ExactMatchMetric + +client.evaluator.metrics.create( + "answer-from-reference", + metric=ExactMatchMetric( + reference="{{reference.expected}}", + candidate="{{sample.output_text}}", + ), +) + +inline_task = AgentEvalTaskInput( + id="capital-france", + intent="Name the capital of France.", + inputs=TaskInputs(instruction="What is the capital of France?"), + reference={"expected": "Paris"}, + metrics=[MetricRef("default/answer-from-reference")], +) +``` + +Stored tasks keep metric references. Inline task metrics are normalized into +content-addressed derived metrics. The stored-task example uses an output-only +metric because stored tasks do not carry the grader-only `reference` field; use +an inline `AgentEvalTaskInput` when held-out per-task data is required. + +## Retrieve, list, and delete + +```python +metric = client.evaluator.metrics.retrieve("answer-exact") +metrics = client.evaluator.metrics.list(metric_type="string-check") +tasks = client.evaluator.tasks.list(page=1, page_size=100, sort="name") +tasksets = client.evaluator.tasksets.list(page=1, page_size=100) + +client.evaluator.tasksets.delete("geography") +client.evaluator.tasks.delete("capital-france") +client.evaluator.metrics.delete("answer-exact") +``` + +Metric listing supports `metric_type` and `include_derived`. Task and taskset +listing support pagination and sorting. Every method accepts an optional +`workspace`; create operations also accept `project`. + +## Query persisted results + +Dataset-driven durable jobs create `eval_results`; agent-evaluation jobs create +`agent_eval_results`. + +```python +row_eval = client.evaluator.eval_results.retrieve("") +row_page = client.evaluator.eval_results.list( + job_id="", + target_kind="model", + target_name="", + dataset_ref="default/eval-data", +) + +agent_eval = client.evaluator.agent_eval_results.retrieve("") +agent_page = client.evaluator.agent_eval_results.list( + job_id="", + target_kind="harbor", + target_name="oracle", +) +``` + +Both result resources support pagination, sorting, workspace override, and +delete. Result indexing is best effort and separate from the authoritative job +artifacts. Retry a short-lived `404`; if the record remains absent, inspect the +job logs and use the artifact bundle. A persisted record is a queryable +summary/index; use the bundle for complete row scores, trials, evidence, and +reports. diff --git a/skills/nemo-evaluator-plugin/references/troubleshooting.md b/skills/nemo-evaluator-plugin/references/troubleshooting.md index e00f8609d0..793342bb28 100644 --- a/skills/nemo-evaluator-plugin/references/troubleshooting.md +++ b/skills/nemo-evaluator-plugin/references/troubleshooting.md @@ -1,38 +1,39 @@ # Evaluation Troubleshooting -The Evaluator plugin CLI surface is `nemo evaluator`. +The plugin CLI surface is `nemo evaluator`. In a repository checkout, prefix +the commands below with `uv run`. -## Quick Checks +## Inspect the installed contracts ```bash -nemo evaluator --help -nemo evaluator evaluate --help +nemo evaluator info +nemo evaluator metric-types nemo evaluator evaluate explain +nemo evaluator agent-evaluate explain ``` -## Local vs Cluster Runs - -Use local execution to validate the spec: - -```bash -nemo evaluator evaluate run --spec-file evaluation-spec.json -``` - -Use cluster submission once the same spec works locally: - -```bash -nemo evaluator evaluate submit \ - --spec-file evaluation-spec.json \ - --workspace default \ - --profile default -``` - -## Common Issues - -| Symptom | Cause | Fix | -|---------|-------|-----| -| `No such command 'evaluation'` | The legacy generated CLI group was removed | Use `nemo evaluator ...` | -| Spec validation error | The submitted spec does not match the plugin schema | Run `nemo evaluator evaluate explain` and update the spec | -| Secret not found during `submit` | The judge metric references a missing NeMo platform secret | Run `nemo secrets list` in the target workspace and create the secret if needed | -| Local `run` cannot authenticate to the judge endpoint | `api_key_secret` points at a NeMo secret name instead of a local environment variable, or the environment variable is unset | Set the API key in the local environment and use that variable name as `api_key_secret`. See [Evaluator API Auth](api-auth.md) | -| Local run works but submit fails | Cluster/profile/workspace configuration issue | Check `nemo evaluator evaluate submit --help`, then retry with explicit `--workspace`, `--profile`, and cluster options | +## Common failures + +| Symptom | Likely cause | Fix | +| --- | --- | --- | +| `No such command 'evaluation'` | The legacy generated CLI group is not the plugin surface | Use `nemo evaluator ...` | +| Spec validation error | Fields do not match the current job schema | Run the matching `explain` command and validate against the spec class before submission | +| Dataset row has missing fields | Jinja templates or `field_mapping` do not match row keys | Inspect one row and every referenced template before rerunning | +| Standalone model/agent authentication fails | `api_key_secret` names a platform secret instead of an environment variable, or the variable is unset | Use an environment-variable name; see [API Auth](api-auth.md) | +| Remote submission returns 409 | The response may describe a missing platform secret, not a duplicate job | Read the response body and verify the workspace secret | +| Built-in metric bundle contains cloudpickle | A legacy or explicit packager was used | Regenerate with `InlineMetricBundlePackager` or the current default | +| Custom metric submission rejects the default packager | Shipping custom code requires explicit opt-in | Pass `HybridMetricBundlePackager()` (preferred) or `CloudpickleMetricBundlePackager()` | +| `ModelRef` fails with the standalone SDK | Model references are resolved by the platform submission path | Use a concrete `Model` with the standalone SDK or use `submit` with `ModelRef` | +| Fileset evaluation cannot load data | The reference, fragment, or workspace is wrong | Verify the `FilesetRef` and access it through the same workspace | +| Result download fails while progress shows 100% | Metric progress finished before the platform job finalized artifacts | Call `job.wait_until_done()` before `get_result()` or `download_artifacts()` | +| Agent-eval rejects the spec | Both or neither of `target` and `trials` were provided | Provide exactly one | +| Taskset evaluation lacks held-out reference data | Stored tasks do not carry grader-only `reference` | Use inline `AgentEvalTaskInput` when the metric needs held-out per-task data | +| Runner target fails to start | The runtime dependency, CLI, config, credentials, or Docker access is missing | Check the selected Codex, Fabric, or Harbor runner prerequisites | + +## Debug in the smallest scope + +1. Validate one expected pass and one expected failure. +2. Inspect row scores or task trials before aggregates. +3. Reproduce metric behavior with the standalone SDK before diagnosing platform infrastructure. +4. For submitted jobs, inspect terminal status and error details. +5. Retry only the failed row, task, or runner configuration when possible. diff --git a/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py index 953454e8e3..623ce22223 100644 --- a/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py +++ b/skills/nemo-evaluator-plugin/scripts/generate_example_specs.py @@ -2,55 +2,220 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Print an exact-match metric bundle example. +"""Generate or check the Evaluator plugin skill's checked JSON specs. -Run from the repo root: +Run from the repository root: - uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py + uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py --check + uv run --frozen python skills/nemo-evaluator-plugin/scripts/generate_example_specs.py --write """ from __future__ import annotations +import argparse import json -import os -import sys +from collections.abc import Callable +from pathlib import Path from typing import Any -DETERMINISTIC_HASH_SEED = "0" -JSON_OUTPUT_INDENT = 4 -SUCCESS_EXIT_CODE = 0 - - -def _ensure_deterministic_hash_seed() -> None: - if os.environ.get("PYTHONHASHSEED") == DETERMINISTIC_HASH_SEED: - return - env = {**os.environ, "PYTHONHASHSEED": DETERMINISTIC_HASH_SEED} - os.execvpe(sys.executable, [sys.executable, *sys.argv], env) +from nemo_evaluator.jobs.agent_spec import AgentEvalInputSpec +from nemo_evaluator.jobs.evaluate import EvaluateInputSpec +from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric +from nemo_evaluator.shared.metric_bundles.inline import InlineMetricBundlePackager +from nemo_evaluator_sdk import ( + ExactMatchMetric, + JSONScoreParser, + LLMJudgeMetric, + Model, + RangeScore, + SecretRef, +) +from nemo_evaluator_sdk.enums import ModelFormat + +SKILL_DIR = Path(__file__).resolve().parents[1] +SPEC_DIR = SKILL_DIR / "assets" / "specs" +SpecBuilder = Callable[[], dict[str, Any]] def _bundle(metric: Any) -> dict[str, Any]: - _ensure_deterministic_hash_seed() - - from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric - from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager - - return bundle_metric(metric, CloudpickleMetricBundlePackager()).model_dump(mode="json") - - -def build_metric_bundle_example() -> dict[str, Any]: - """Return bundled JSON for one configured SDK metric.""" - from nemo_evaluator_sdk.metrics.exact_match import ExactMatchMetric - - metric = ExactMatchMetric( - reference="{{item.gold_answer}}", - candidate="{{item.prediction}}", + return bundle_metric(metric, InlineMetricBundlePackager()).model_dump(mode="json") + + +def build_exact_match_spec() -> dict[str, Any]: + """Return a two-row offline exact-match spec.""" + return { + "metrics": [ + _bundle( + ExactMatchMetric( + reference="{{item.expected}}", + candidate="{{item.output}}", + ) + ) + ], + "dataset": [ + {"expected": "Paris", "output": "Paris"}, + {"expected": "Paris", "output": "London"}, + ], + "params": {"parallelism": 2}, + } + + +def build_llm_as_judge_spec() -> dict[str, Any]: + """Return a minimal online generation plus LLM-judge spec.""" + model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/nemotron-3-super-120b-a12b", + api_key_secret=SecretRef(root="NVIDIA_API_KEY"), + format=ModelFormat.NVIDIA_NIM, ) - return _bundle(metric) - - -def main() -> int: - print(json.dumps(build_metric_bundle_example(), indent=JSON_OUTPUT_INDENT)) - return SUCCESS_EXIT_CODE + judge = LLMJudgeMetric( + model=model, + scores=[ + RangeScore( + name="helpfulness", + description="How well the response helps the user.", + minimum=0, + maximum=4, + parser=JSONScoreParser(json_path="helpfulness"), + ) + ], + prompt_template={ + "messages": [ + { + "role": "system", + "content": 'Rate helpfulness from 0-4. Return JSON only: {"helpfulness": }.', + }, + { + "role": "user", + "content": "Request: {{item.input}}\nResponse: {{sample.output_text}}", + }, + ] + }, + ) + return { + "metrics": [_bundle(judge)], + "dataset": [ + {"input": "What is the capital of France?"}, + {"input": "How do I make scrambled eggs?"}, + ], + "params": { + "parallelism": 2, + "limit_samples": 2, + "request_timeout": 120, + "max_retries": 3, + }, + "target": model.model_dump(mode="json"), + "prompt_template": { + "messages": [{"role": "user", "content": "{{item.input}}"}], + }, + } + + +def build_fabric_agent_eval_spec() -> dict[str, Any]: + """Return a one-task durable Fabric agent-evaluation spec.""" + return { + "tasks": [ + { + "id": "capital-france", + "intent": "Name the capital of France.", + "inputs": {"instruction": "What is the capital of France?"}, + "reference": {"expected": "Paris"}, + "metrics": [ + _bundle( + ExactMatchMetric( + reference="{{reference.expected}}", + candidate="{{sample.output_text}}", + ) + ) + ], + } + ], + "target": { + "kind": "fabric", + "config": { + "metadata": {"name": "readme-fabric-smoke"}, + "harness": {"adapter_id": "nvidia.fabric.codex"}, + }, + "model": "/", + "capture_trajectory": False, + }, + "max_concurrent_tasks": 1, + "fail_fast": True, + "benchmark": {"name": "readme-fabric-smoke"}, + } + + +SPEC_BUILDERS: dict[str, SpecBuilder] = { + "exact_match_metric.json": build_exact_match_spec, + "llm_as_judge.json": build_llm_as_judge_spec, +} + +AGENT_SPEC_BUILDERS: dict[str, SpecBuilder] = { + "fabric_agent_eval.json": build_fabric_agent_eval_spec, +} + + +def generated_specs() -> dict[Path, dict[str, Any]]: + """Build and validate every checked dataset-evaluation spec.""" + specs: dict[Path, dict[str, Any]] = {} + for name, builder in SPEC_BUILDERS.items(): + payload = builder() + EvaluateInputSpec.model_validate(payload) + specs[SPEC_DIR / name] = payload + return specs + + +def generated_agent_specs() -> dict[Path, dict[str, Any]]: + """Build and validate every checked agent-evaluation spec.""" + specs: dict[Path, dict[str, Any]] = {} + for name, builder in AGENT_SPEC_BUILDERS.items(): + payload = builder() + AgentEvalInputSpec.model_validate(payload) + specs[SPEC_DIR / name] = payload + return specs + + +def _all_generated_specs() -> dict[Path, dict[str, Any]]: + return {**generated_specs(), **generated_agent_specs()} + + +def _render(payload: dict[str, Any]) -> str: + return json.dumps(payload, indent=2) + "\n" + + +def write_specs() -> int: + """Write generated specs to their checked locations.""" + for path, payload in _all_generated_specs().items(): + path.write_text(_render(payload), encoding="utf-8") + print(f"wrote {path.relative_to(SKILL_DIR)}") + return 0 + + +def check_specs() -> int: + """Return nonzero when a checked spec differs from generated output.""" + stale: list[Path] = [] + for path, payload in _all_generated_specs().items(): + if not path.is_file() or path.read_text(encoding="utf-8") != _render(payload): + stale.append(path) + + if stale: + for path in stale: + print(f"out of date: {path.relative_to(SKILL_DIR)}") + print("run with --write to refresh the checked specs") + return 1 + + count = len(SPEC_BUILDERS) + len(AGENT_SPEC_BUILDERS) + print(f"{count} evaluator example specs are up to date") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--check", action="store_true", help="Check generated specs without writing.") + action.add_argument("--write", action="store_true", help="Write generated specs.") + args = parser.parse_args(argv) + return check_specs() if args.check else write_specs() if __name__ == "__main__": From 1742e09445be1bafe6cc16893216905b66edc322 Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Mon, 3 Aug 2026 16:46:38 -0700 Subject: [PATCH 2/4] chore: cleanup Signed-off-by: Nick Goncharenko --- docs/evaluator/agent-eval/index.mdx | 4 ++-- docs/evaluator/index.mdx | 4 ++-- skills/nemo-evaluator-plugin/references/evaluation-shapes.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/evaluator/agent-eval/index.mdx b/docs/evaluator/agent-eval/index.mdx index ccd50d8f53..edf4f99a13 100644 --- a/docs/evaluator/agent-eval/index.mdx +++ b/docs/evaluator/agent-eval/index.mdx @@ -13,8 +13,8 @@ got there*. Each task carries its own metrics, so a single suite can grade heter -**Local and platform durable interfaces are available.** -- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +**Local and platform durable interfaces are available.** +- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 7d247118ff..849e6545bf 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -120,8 +120,8 @@ result = job.get_result() -**Agent evaluation has local and platform durable interfaces.** -- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +**Agent evaluation has local and platform durable interfaces.** +- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces diff --git a/skills/nemo-evaluator-plugin/references/evaluation-shapes.md b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md index 494fa45b97..6d04edc11b 100644 --- a/skills/nemo-evaluator-plugin/references/evaluation-shapes.md +++ b/skills/nemo-evaluator-plugin/references/evaluation-shapes.md @@ -24,7 +24,7 @@ Each row contains the fields consumed by the metric, for example: ``` Read [SDK Execution](execution.md) for datasets, targets, field mappings, -submission, and results. +submission, and results. Read [Metric Selection](metric-selection.md) to choose the scorer. From 8b67ab17bd3b69c50ed0f42fc4bee487d36c637e Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Mon, 3 Aug 2026 20:12:42 -0700 Subject: [PATCH 3/4] chore: apply fixes based on comments Signed-off-by: Nick Goncharenko --- .cursor/hooks/state/continual-learning.json | 8 ++++ docs/evaluator/agent-eval/index.mdx | 4 +- docs/evaluator/index.mdx | 4 +- .../agent_eval/test_fabric_integration.py | 3 +- plugins/nemo-evaluator/README.md | 6 ++- .../tests/test_agent_evaluate.py | 39 ++++++++++++++++++- .../nemo-evaluator/tests/test_evaluate_job.py | 27 +++++++++++-- .../tests/test_skill_examples.py | 11 +++++- script/dev-install-fabric.sh | 5 ++- skills/nemo-evaluator-plugin/SKILL.md | 9 ++++- .../references/agent-evaluation.md | 2 +- .../references/execution.md | 19 ++++++++- .../references/metric-selection.md | 6 +-- .../references/resources.md | 3 ++ .../references/troubleshooting.md | 1 + 15 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 .cursor/hooks/state/continual-learning.json diff --git a/.cursor/hooks/state/continual-learning.json b/.cursor/hooks/state/continual-learning.json new file mode 100644 index 0000000000..3ec324fa81 --- /dev/null +++ b/.cursor/hooks/state/continual-learning.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "lastRunAtMs": 0, + "turnsSinceLastRun": 2, + "lastTranscriptMtimeMs": null, + "lastProcessedGenerationId": "4982354d-87b6-4cb7-bd05-44f5f77bec33", + "trialStartedAtMs": null +} diff --git a/docs/evaluator/agent-eval/index.mdx b/docs/evaluator/agent-eval/index.mdx index edf4f99a13..b095bdcf01 100644 --- a/docs/evaluator/agent-eval/index.mdx +++ b/docs/evaluator/agent-eval/index.mdx @@ -13,8 +13,8 @@ got there*. Each task carries its own metrics, so a single suite can grade heter -**Local and platform durable interfaces are available.** -- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +**Local interfaces and durable platform interfaces are available.** +- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 849e6545bf..c21494fb3f 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -120,8 +120,8 @@ result = job.get_result() -**Agent evaluation has local and platform durable interfaces.** -- Use `AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs +**Agent evaluation has local interfaces and durable platform interfaces.** +- Use `await AgentEvaluator().run(tasks=..., target=...)` for local task-driven SDK runs that do not require running nemo-platform. - Use the Evaluator plugin's `uv run nemo evaluator agent-evaluate submit` job for durable runs with inline tasks or stored tasksets. The high-level `client.evaluator.run/submit` interfaces diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py index 68f7da3e40..fd75eca7f8 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py @@ -251,7 +251,8 @@ def _codex_adapter_installed() -> bool: requires_live_fabric = pytest.mark.skipif( not _LIVE_READY, reason=( - "needs the harness adapters (uv sync --package nemo-evaluator-sdk --extra fabric) + the nemo-relay gateway " + "needs the harness adapters " + "(uv sync --frozen --package nemo-evaluator-sdk --extra fabric --inexact) + the nemo-relay gateway " "(script/dev-install-fabric.sh) + codex on PATH" ), ) diff --git a/plugins/nemo-evaluator/README.md b/plugins/nemo-evaluator/README.md index 334c55854d..0191df4808 100644 --- a/plugins/nemo-evaluator/README.md +++ b/plugins/nemo-evaluator/README.md @@ -71,8 +71,9 @@ nemo jobs results download row-scores --job --output-file row-scores. See also the checked LLM-judge spec example in `skills/nemo-evaluator-plugin/assets/specs/llm_as_judge.json`. -### Plugin SDK Execution -Use the mounted SDK resource for local and durable execution: +### Platform SDK Execution + +Use the mounted SDK resource to submit durable evaluation jobs: ```python from nemo_evaluator_sdk import ExactMatchMetric, RunConfig @@ -98,6 +99,7 @@ job.wait_until_done() remote_result = job.get_result() artifact_dir = job.download_artifacts("evaluation-artifacts") ``` + `submit` returns an `EvaluatorJobResource`. Always call `wait_until_done()` before retrieving result artifacts. diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index 8c03b81a34..36b7f0260a 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json from collections.abc import Sequence from pathlib import Path from typing import Any, cast @@ -31,7 +32,7 @@ Target, ) from nemo_evaluator.metric_refs import MetricRef -from nemo_evaluator.shared.metric_bundles.bundles import bundle_metric +from nemo_evaluator.shared.metric_bundles.bundles import MetricBundle, bundle_metric from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager from nemo_evaluator.tasks.agent_evaluate import main as agent_eval_task_main from nemo_evaluator.tasks.runner import SDK_INITIALIZATION_EXIT_CODE @@ -66,6 +67,10 @@ def _inline_metric() -> MetricInline: return MetricInline.model_validate(bundle.model_dump(mode="json")) +def _repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + def _task_spec() -> AgentEvalTaskSpec: return AgentEvalTaskSpec( id="task-1", @@ -436,6 +441,38 @@ def _assert_agent_eval_step_entrypoint(job_spec: PlatformJobSpec) -> None: assert container.command == ["nemo_evaluator.tasks.agent_evaluate"] +async def test_checked_fabric_spec_transforms_and_compiles() -> None: + path = _repo_root() / "skills/nemo-evaluator-plugin/assets/specs/fabric_agent_eval.json" + payload = json.loads(path.read_text(encoding="utf-8")) + input_spec = AgentEvalInputSpec.model_validate(payload) + + spec = await AgentEvalJob.to_spec( + input_spec, + workspace="default", + entity_client=None, + async_sdk=None, + is_local=False, + ) + + assert isinstance(spec, AgentEvalSpec) + assert isinstance(spec.tasks, list) + bundle = MetricBundle.model_validate(spec.tasks[0].metrics[0].model_dump(mode="json")) + assert bundle.payload.kind == "inline" + + compiled = await AgentEvalJob.compile( + workspace="default", + spec=spec, + entity_client=None, + job_name=None, + async_sdk=None, + ) + job_spec = PlatformJobSpec.model_validate(compiled) + _assert_agent_eval_step_entrypoint(job_spec) + config = cast(dict[str, Any], job_spec.steps[0].config) + assert config["target"]["kind"] == "fabric" + assert config["tasks"][0]["metrics"][0]["payload"]["kind"] == "inline" + + @pytest.mark.parametrize( ("target", "expected_kind", "expected_endpoint_name"), [ diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index a321deb950..b37a5bfde8 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -15,7 +15,6 @@ import pytest from nemo_evaluator.cli import EvaluatorPluginCLI from nemo_evaluator.filesets import FilesetRef -from nemo_evaluator.jobs.compiler import compile_evaluate_job from nemo_evaluator.jobs.evaluate import ( AGGREGATE_SCORES_RESULT_NAME, ARTIFACTS_RESULT_NAME, @@ -275,6 +274,12 @@ def test_checked_in_example_spec_uses_inline_metric_bundle_shape(spec_path: Path for metric_payload in payload["metrics"]: bundle = MetricBundle.model_validate(metric_payload) assert bundle.payload.kind == "inline" + assert { + "python_version", + "cloudpickle_version", + "pickle_protocol", + "blob", + }.isdisjoint(metric_payload["payload"]) assert bundle.metric_type == metric_payload["metric_type"] assert unbundle_metric(bundle).type == bundle.metric_type @@ -283,11 +288,25 @@ def test_checked_in_example_spec_uses_inline_metric_bundle_shape(spec_path: Path "spec_path", EXAMPLE_SPEC_PATHS, ) -def test_checked_in_example_spec_compiles(spec_path: Path) -> None: +async def test_checked_in_example_spec_transforms_and_compiles(spec_path: Path) -> None: payload = json.loads((_repo_root() / spec_path).read_text(encoding="utf-8")) - spec = EvaluateSpec.model_validate(payload) - compiled = compile_evaluate_job(spec) + input_spec = EvaluateInputSpec.model_validate(payload) + spec = await EvaluateJob.to_spec( + input_spec, + workspace="default", + entity_client=None, + async_sdk=None, + is_local=False, + ) + assert isinstance(spec, EvaluateSpec) + compiled = await EvaluateJob.compile( + workspace="default", + spec=spec, + entity_client=None, + job_name=None, + async_sdk=None, + ) assert "metric" not in payload assert len(spec.metrics) >= 1 diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py index f9fffb4aee..6cee080505 100644 --- a/plugins/nemo-evaluator/tests/test_skill_examples.py +++ b/plugins/nemo-evaluator/tests/test_skill_examples.py @@ -160,7 +160,16 @@ def test_checked_durable_fabric_job_is_a_valid_agent_eval_spec() -> None: spec = AgentEvalInputSpec.model_validate(generated[path]) assert isinstance(spec.tasks, list) - metric = unbundle_metric(MetricBundle.model_validate(spec.tasks[0].metrics[0].model_dump(mode="json"))) + metric_payload = spec.tasks[0].metrics[0].model_dump(mode="json") + bundle = MetricBundle.model_validate(metric_payload) + assert bundle.payload.kind == "inline" + assert { + "python_version", + "cloudpickle_version", + "pickle_protocol", + "blob", + }.isdisjoint(metric_payload["payload"]) + metric = unbundle_metric(bundle) assert isinstance(metric, ExactMatchMetric) assert isinstance(spec.target, FabricRunnerTarget) assert spec.target.capture_trajectory is False diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh index c008ec9dbf..5443e74178 100755 --- a/script/dev-install-fabric.sh +++ b/script/dev-install-fabric.sh @@ -17,8 +17,9 @@ # # To run against an unreleased Fabric instead of the locked wheels, install the checkout directly: # uv pip install --python .venv/bin/python "/path/to/NeMo-Fabric[codex,relay,runtime]" -# and use the package-scoped command above to get back to the locked state. (That needs cargo — -# Fabric builds a Rust/pyo3 extension from source.) +# and use the package-scoped command above to restore the locked project dependencies without +# pruning unrelated installed packages. (That needs cargo — Fabric builds a Rust/pyo3 extension +# from source.) # # A live codex run additionally needs the `codex` CLI + `codex login` auth. # See skills/nemo-evaluator-plugin/references/agent-evaluation.md and diff --git a/skills/nemo-evaluator-plugin/SKILL.md b/skills/nemo-evaluator-plugin/SKILL.md index 19643954a8..27ae26ae52 100644 --- a/skills/nemo-evaluator-plugin/SKILL.md +++ b/skills/nemo-evaluator-plugin/SKILL.md @@ -36,8 +36,11 @@ configuration, field mapping, job lifecycle, and custom metric packaging. result queries. ## CLI Interface + ### Prerequisites -All commands in this file assume that the shell's working dir is at the root of the Nvidia-NeMo/nemo-platform repo + +All commands in this file assume that the shell's working directory is the root +of the NVIDIA-NeMo/nemo-platform repository. In a NeMo Platform repository checkout, run commands through the workspace: @@ -76,11 +79,14 @@ artifacts; metric progress can reach 100 percent before the platform job is terminal. ## Task-driven agent evaluation examples + **Standalone SDK evaluation** + Use `AgentEvaluator().run(...)` for standalone task-driven SDK evaluation. Its `target` can be a `Model`, a `GenericAgent`, or a direct `AgentTaskRunner`. **Platform job evaluation** + Use the plugin `agent-evaluate submit` job for platform task evaluation. Its target is a `ModelTarget`, `AgentTarget`, `CodexRunnerTarget`, `FabricRunnerTarget`, or `HarborRunnerTarget`; alternatively provide @@ -118,5 +124,6 @@ optional adapters. authentication, job, result, or runner behavior fails. ## Follow security best practices + Never print, serialize, or commit secret values. Store only environment-variable names or platform secret references in specs and examples. diff --git a/skills/nemo-evaluator-plugin/references/agent-evaluation.md b/skills/nemo-evaluator-plugin/references/agent-evaluation.md index d9dd143909..d528ddb6c2 100644 --- a/skills/nemo-evaluator-plugin/references/agent-evaluation.md +++ b/skills/nemo-evaluator-plugin/references/agent-evaluation.md @@ -5,7 +5,7 @@ Read this file for agentic task-driven evaluation, direct SDK runners, platform ## Choose standalone SDK or platform job -Use `AgentEvaluator` lightweight in-process evaluation that does not require a running nemo-platform: +Use `AgentEvaluator` for lightweight in-process evaluation that does not require a running nemo-platform: ```python from nemo_evaluator_sdk.agent_eval.evaluator import AgentEvaluator diff --git a/skills/nemo-evaluator-plugin/references/execution.md b/skills/nemo-evaluator-plugin/references/execution.md index f4842417e9..fdf5ac719f 100644 --- a/skills/nemo-evaluator-plugin/references/execution.md +++ b/skills/nemo-evaluator-plugin/references/execution.md @@ -9,6 +9,7 @@ prefix them with `uv run`. ## Validate standalone, then submit to the platform ### Standalone SDK + Use the standalone SDK for the fastest in-process metric loop: ```python @@ -29,14 +30,16 @@ print(result.aggregate_scores) ``` **Platform CLI** + Platform CLI equivalent for the same checked metric and rows: ```bash -nemo evaluator evaluate submit \ +uv run nemo evaluator evaluate submit \ --spec-file skills/nemo-evaluator-plugin/assets/specs/exact_match_metric.json ``` **Platform Python SDK** + Use `client.evaluator.submit` for execution through the installed nemo-evaluator-plugin: ```python @@ -61,11 +64,13 @@ result = job.get_result() ``` ### Use a Fileset for the dataset + The job will run in the container environment. The plugin submission dataset accepts inline rows, a `str` or `Path`, and `FilesetRef`: **Platform SDK** + ```python from nemo_evaluator.sdk import FilesetRef @@ -73,7 +78,9 @@ dataset = FilesetRef("default/eval-data") ``` **Platform CLI** + CLI equivalent, using a stored metric and fileset: + ```bash nemo evaluator evaluate submit \ --spec '{"metrics":["default/exact-answer"],"dataset":"default/eval-data"}' @@ -86,6 +93,7 @@ Provide a prompt template when the evaluator should generate output before scoring. **Platform SDK** + ```python from nemo_evaluator_sdk import ExactMatchMetric, Model, RunConfigOnlineModel @@ -127,6 +135,7 @@ Use `FieldMapping` when the metric expects canonical evaluator fields but the dataset uses different column names: **Platform SDK** + ```python from nemo_evaluator_sdk import FieldMapping @@ -135,9 +144,11 @@ mapping = FieldMapping( reference="gold_answer", ) ``` + Pass it as `field_mapping=mapping` when submitting the job. **Platform CLI** + ```bash nemo evaluator evaluate submit --spec \ '{ @@ -155,6 +166,7 @@ Use field_mapping when a metric or online prompt uses canonical evaluator fields ## Getting job results **Platform SDK** + ```python job = client.evaluator.submit( metric=metric, @@ -168,7 +180,9 @@ job.wait_until_done() result = job.get_result() artifacts = job.download_artifacts("evaluation-artifacts") ``` + `EvaluatorJobResource` also exposes methods for job lifecycle management: + - `name` and `job` - `get_job_status()` - `check_if_complete(raise_if_not_complete=False)` @@ -176,6 +190,7 @@ artifacts = job.download_artifacts("evaluation-artifacts") - `as_async()` **Platform CLI** + Poll until the job is completed before downloading results: ```bash @@ -194,12 +209,12 @@ group. Use the SDK submission handle when the workflow needs those lifecycle operations. ### Notes + - Always wait for terminal completion. A metric can report 100 percent progress before the platform finishes publishing result artifacts. - `submit` accepts a concrete `Model` or `ModelRef`; the platform resolves model references in the target workspace. - ## Multiple metrics The high-level plugin helper takes one runtime `metric`. To combine metrics, diff --git a/skills/nemo-evaluator-plugin/references/metric-selection.md b/skills/nemo-evaluator-plugin/references/metric-selection.md index 9e521e8311..4f5647460c 100644 --- a/skills/nemo-evaluator-plugin/references/metric-selection.md +++ b/skills/nemo-evaluator-plugin/references/metric-selection.md @@ -31,8 +31,8 @@ timeout prerequisites. List current metric names and inspect one schema: ```bash -nemo evaluator metric-types -nemo evaluator metric-types exact-match +uv run nemo evaluator metric-types +uv run nemo evaluator metric-types exact-match ``` ## Validate the mapping @@ -73,7 +73,7 @@ result = Evaluator().run_sync(metrics=[accuracy, style], dataset=rows) Platform job — put multiple stored metrics on the job spec: ```bash -nemo evaluator evaluate submit --spec \ +uv run nemo evaluator evaluate submit --spec \ '{"metrics":["default/accuracy","default/style"],"dataset":"default/eval-data"}' ``` diff --git a/skills/nemo-evaluator-plugin/references/resources.md b/skills/nemo-evaluator-plugin/references/resources.md index 2505c43e2e..826b02794e 100644 --- a/skills/nemo-evaluator-plugin/references/resources.md +++ b/skills/nemo-evaluator-plugin/references/resources.md @@ -27,6 +27,9 @@ from nemo_evaluator.api.schemas import ( TasksetInput, ) from nemo_evaluator_sdk import StringCheckMetric +from nemo_platform import NeMoPlatform + +client = NeMoPlatform(base_url="", workspace="") client.evaluator.metrics.create( "answer-exact", diff --git a/skills/nemo-evaluator-plugin/references/troubleshooting.md b/skills/nemo-evaluator-plugin/references/troubleshooting.md index 793342bb28..ed7348e189 100644 --- a/skills/nemo-evaluator-plugin/references/troubleshooting.md +++ b/skills/nemo-evaluator-plugin/references/troubleshooting.md @@ -22,6 +22,7 @@ nemo evaluator agent-evaluate explain | Standalone model/agent authentication fails | `api_key_secret` names a platform secret instead of an environment variable, or the variable is unset | Use an environment-variable name; see [API Auth](api-auth.md) | | Remote submission returns 409 | The response may describe a missing platform secret, not a duplicate job | Read the response body and verify the workspace secret | | Built-in metric bundle contains cloudpickle | A legacy or explicit packager was used | Regenerate with `InlineMetricBundlePackager` or the current default | +| `cloudpickle metric payload was created with Python ...` (HTTP 422) | The bundle was created with a different Python major/minor runtime | For a built-in metric, regenerate the checked inline JSON spec; for an intentional custom metric, recreate the bundle with the worker's Python major/minor version | | Custom metric submission rejects the default packager | Shipping custom code requires explicit opt-in | Pass `HybridMetricBundlePackager()` (preferred) or `CloudpickleMetricBundlePackager()` | | `ModelRef` fails with the standalone SDK | Model references are resolved by the platform submission path | Use a concrete `Model` with the standalone SDK or use `submit` with `ModelRef` | | Fileset evaluation cannot load data | The reference, fragment, or workspace is wrong | Verify the `FilesetRef` and access it through the same workspace | From d97623d98e311832603e7c3a14713cb783093eeb Mon Sep 17 00:00:00 2001 From: Nick Goncharenko Date: Mon, 3 Aug 2026 20:33:49 -0700 Subject: [PATCH 4/4] chore: delete unintended change Signed-off-by: Nick Goncharenko --- .cursor/hooks/state/continual-learning.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .cursor/hooks/state/continual-learning.json diff --git a/.cursor/hooks/state/continual-learning.json b/.cursor/hooks/state/continual-learning.json deleted file mode 100644 index 3ec324fa81..0000000000 --- a/.cursor/hooks/state/continual-learning.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "version": 1, - "lastRunAtMs": 0, - "turnsSinceLastRun": 2, - "lastTranscriptMtimeMs": null, - "lastProcessedGenerationId": "4982354d-87b6-4cb7-bd05-44f5f77bec33", - "trialStartedAtMs": null -}