diff --git a/CHANGELOG.md b/CHANGELOG.md index 03b906b..1f6a7eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Contributors add user-facing entries under `[Unreleased]` in the same PR. Mainta ## [Unreleased] +### Added + +- **Tests**: `tests/test_registry_docs.py` — CI doc-drift guards that verify skill catalog, examples index, and agent-loops reference matrix stay in sync with the registry (#183). + ## [0.3.9] - 2026-06-26 ### Fixed diff --git a/docs/TESTING.md b/docs/TESTING.md index db14d10..f021b67 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -56,6 +56,7 @@ pip install -r requirements.txt - Core engine health: loader, CLI, issuer rules, version policy. - `tests/test_skill_issuer.py` also enforces registry packaging (`__init__.py`), issuer metadata, and presence of `test_skill.py` in every skill bundle. +- `tests/test_registry_docs.py` enforces doc-drift parity: skill catalog index matches manifests, examples README matches scripts on disk, and agent-loops.md references every registered skill. - Lives at the **root of `tests/`** only (`tests/test_loader.py`, `tests/test_cli.py`, …). - Clone-repo only; runs in CI via `pytest tests/` together with maintainer tests below. @@ -77,7 +78,7 @@ pip install -r requirements.txt | :--- | :--- | :--- | | Manifest + execute contract for one skill | Bundle test | `skills/compliance/tos_evaluator/test_skill.py` | | Loader path + mocked externals (optional depth) | Maintainer test | `tests/skills/compliance/test_tos_evaluator.py` | -| Loader, CLI, registry issuer rules | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py` | +| Loader, CLI, registry issuer rules | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py`, `tests/test_registry_docs.py` | | End-to-end provider demo script | Usage example | `examples/gemini_tos_evaluator.py` | **Rule of thumb:** if it ships with the skill and must pass before merge → **bundle test** (CI + local). If it is extra regression depth for clone-repo work → **maintainer test** (optional). If it proves provider integration → **example**, not pytest. diff --git a/docs/usage/agent_loops.md b/docs/usage/agent_loops.md index 60b5f42..19a1ca5 100644 --- a/docs/usage/agent_loops.md +++ b/docs/usage/agent_loops.md @@ -67,3 +67,5 @@ skills in one harness. | `data_engineering/novelty_extractor` | `novelty_extractor_demo.py` (local execute) | `gemini_novelty_extractor.py` | (catalog page) | (catalog page) | (catalog page) | `ollama_novelty_extractor.py` | | `dev_tools/issue_resolver` | - | `gemini_issue_resolver.py` | `claude_issue_resolver.py` | (catalog page) | (catalog page) | `ollama_issue_resolver.py` | | `wellness/mental_coach` | `mental_coach_demo.py` (local execute) | (catalog page) | (catalog page) | (catalog page) | (catalog page) | (catalog page) | +| `defi/evm_tx_handler` | - | `gemini_evm_tx_handler.py` | `claude_evm_tx_handler.py` | - | - | - | + diff --git a/tests/test_registry_docs.py b/tests/test_registry_docs.py new file mode 100644 index 0000000..5b3a754 --- /dev/null +++ b/tests/test_registry_docs.py @@ -0,0 +1,166 @@ +"""Repository documentation consistency tests.""" + +from pathlib import Path +import re + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + +SKILL_PATTERN = re.compile(r"`([\w-]+/[\w-]+)`") +EXAMPLE_PATTERN = re.compile(r"^\|[^|]*`([\w-]+\.py)`", re.MULTILINE) + +# TODO: script examples/issue_resolver_github_context.py is a shared helper module and +# should be renamed to examples/issue_resolver_common.py (see #183). +GRANDFATHERED_EXAMPLES: set[str] = {"issue_resolver_github_context.py"} + + +def get_manifested_skills(skills_root: Path) -> set[str]: + """Return all skill IDs containing a manifest.yaml.""" + return { + path.parent.relative_to(skills_root).as_posix() + for path in skills_root.rglob("manifest.yaml") + } + + +def get_cataloged_skills(readme: Path) -> set[str]: + """Return all skill IDs listed in docs/skills/README.md.""" + return set(SKILL_PATTERN.findall(readme.read_text(encoding="utf-8"))) + + +@pytest.fixture(scope="session") +def manifested_skills() -> set[str]: + return get_manifested_skills(REPO_ROOT / "skills") + + +@pytest.fixture(scope="session") +def cataloged_skills() -> set[str]: + return get_cataloged_skills(REPO_ROOT / "docs" / "skills" / "README.md") + + +@pytest.fixture(scope="session") +def example_scripts() -> set[str]: + examples_dir = REPO_ROOT / "examples" + return { + path.name + for path in examples_dir.glob("*.py") + if not path.name.endswith("_common.py") + and path.name not in GRANDFATHERED_EXAMPLES + } + + +@pytest.fixture(scope="session") +def indexed_example_scripts() -> set[str]: + readme = (REPO_ROOT / "examples" / "README.md").read_text(encoding="utf-8") + return set(EXAMPLE_PATTERN.findall(readme)) + + +@pytest.fixture(scope="session") +def agent_loops_text() -> str: + return ( + (REPO_ROOT / "docs" / "usage" / "agent_loops.md") + .read_text(encoding="utf-8") + .lower() + ) + + +def test_readme_matches_manifests( + cataloged_skills: set[str], + manifested_skills: set[str], +): + """README skill index matches manifested skills.""" + + missing = manifested_skills - cataloged_skills + extra = cataloged_skills - manifested_skills + + assert ( + not missing + ), "Skills with manifest but missing from docs/skills/README.md:\n" + "\n".join( + f" - {skill}" for skill in sorted(missing) + ) + + assert ( + not extra + ), "Skills listed in docs/skills/README.md without a manifest.yaml:\n" + "\n".join( + f" - {skill}" for skill in sorted(extra) + ) + + +def test_manifested_skills_have_catalog_pages( + manifested_skills: set[str], +): + """Every manifested skill has a catalog page.""" + + docs_root = REPO_ROOT / "docs" / "skills" + + missing = [ + skill + for skill in sorted(manifested_skills) + if not (docs_root / f"{Path(skill).name}.md").exists() + ] + + assert not missing, "Missing catalog pages:\n" + "\n".join( + f" - docs/skills/{Path(skill).name}.md ({skill})" for skill in missing + ) + + +def test_catalog_pages_have_manifests( + manifested_skills: set[str], +): + """Every catalog page corresponds to a manifested skill.""" + + docs_root = REPO_ROOT / "docs" / "skills" + + expected = {Path(skill).name for skill in manifested_skills} + + actual = {page.stem for page in docs_root.glob("*.md") if page.name != "README.md"} + + orphaned = actual - expected + + assert not orphaned, "Catalog pages without a matching manifest:\n" + "\n".join( + f" - docs/skills/{page}.md" for page in sorted(orphaned) + ) + + +def test_examples_readme_matches_files( + example_scripts: set[str], + indexed_example_scripts: set[str], +): + """Examples README matches runnable scripts.""" + + missing = example_scripts - indexed_example_scripts + orphaned = indexed_example_scripts - example_scripts + + assert ( + not missing + ), "Example scripts missing from examples/README.md:\n" + "\n".join( + f" - {script}" for script in sorted(missing) + ) + + assert ( + not orphaned + ), "README references non-existent example scripts:\n" + "\n".join( + f" - {script}" for script in sorted(orphaned) + ) + + +def test_agent_loops_reference_all_skills( + manifested_skills: set[str], + agent_loops_text: str, +): + """Every manifested skill is referenced in agent_loops.md.""" + + missing = [] + + for skill in sorted(manifested_skills): + skill_name = Path(skill).name + + if ( + skill.lower() not in agent_loops_text + and skill_name.lower() not in agent_loops_text + ): + missing.append(skill) + + assert not missing, "Skills missing from docs/usage/agent_loops.md:\n" + "\n".join( + f" - {skill}" for skill in missing + )