Add documentation consistency tests

CHANGELOG.md

@@ -8,6 +8,10 @@ Contributors add user-facing entries under `[Unreleased]` in the same PR. Mainta
 
 ## [Unreleased]
 
+### Added
+
+- **Tests**: `tests/test_registry_docs.py` — CI doc-drift guards that verify skill catalog, examples index, and agent-loops reference matrix stay in sync with the registry (#183).
+
 ## [0.3.9] - 2026-06-26
 
 ### Fixed

docs/TESTING.md

@@ -56,6 +56,7 @@ pip install -r requirements.txt
 
 - Core engine health: loader, CLI, issuer rules, version policy.
 - `tests/test_skill_issuer.py` also enforces registry packaging (`__init__.py`), issuer metadata, and presence of `test_skill.py` in every skill bundle.
+- `tests/test_registry_docs.py` enforces doc-drift parity: skill catalog index matches manifests, examples README matches scripts on disk, and agent-loops.md references every registered skill.
 - Lives at the **root of `tests/`** only (`tests/test_loader.py`, `tests/test_cli.py`, …).
 - Clone-repo only; runs in CI via `pytest tests/` together with maintainer tests below.
 
@@ -77,7 +78,7 @@ pip install -r requirements.txt
 | :--- | :--- | :--- |
 | Manifest + execute contract for one skill | Bundle test | `skills/compliance/tos_evaluator/test_skill.py` |
 | Loader path + mocked externals (optional depth) | Maintainer test | `tests/skills/compliance/test_tos_evaluator.py` |
-| Loader, CLI, registry issuer rules | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py` |
+| Loader, CLI, registry issuer rules | Framework test | `tests/test_loader.py`, `tests/test_skill_issuer.py`, `tests/test_registry_docs.py` |
 | End-to-end provider demo script | Usage example | `examples/gemini_tos_evaluator.py` |
 
 **Rule of thumb:** if it ships with the skill and must pass before merge → **bundle test** (CI + local). If it is extra regression depth for clone-repo work → **maintainer test** (optional). If it proves provider integration → **example**, not pytest.

docs/usage/agent_loops.md

@@ -67,3 +67,5 @@ skills in one harness.
 | `data_engineering/novelty_extractor` | `novelty_extractor_demo.py` (local execute) | `gemini_novelty_extractor.py` | (catalog page) | (catalog page) | (catalog page) | `ollama_novelty_extractor.py` |
 | `dev_tools/issue_resolver` | - | `gemini_issue_resolver.py` | `claude_issue_resolver.py` | (catalog page) | (catalog page) | `ollama_issue_resolver.py` |
 | `wellness/mental_coach` | `mental_coach_demo.py` (local execute) | (catalog page) | (catalog page) | (catalog page) | (catalog page) | (catalog page) |
+| `defi/evm_tx_handler` | - | `gemini_evm_tx_handler.py` | `claude_evm_tx_handler.py` | - | - | - |
+

tests/test_registry_docs.py

@@ -0,0 +1,166 @@
+"""Repository documentation consistency tests."""
+
+from pathlib import Path
+import re
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+SKILL_PATTERN = re.compile(r"`([\w-]+/[\w-]+)`")
+EXAMPLE_PATTERN = re.compile(r"^\|[^|]*`([\w-]+\.py)`", re.MULTILINE)
+
+# TODO: script examples/issue_resolver_github_context.py is a shared helper module and
+#       should be renamed to examples/issue_resolver_common.py (see #183).
+GRANDFATHERED_EXAMPLES: set[str] = {"issue_resolver_github_context.py"}
+
+
+def get_manifested_skills(skills_root: Path) -> set[str]:
+    """Return all skill IDs containing a manifest.yaml."""
+    return {
+        path.parent.relative_to(skills_root).as_posix()
+        for path in skills_root.rglob("manifest.yaml")
+    }
+
+
+def get_cataloged_skills(readme: Path) -> set[str]:
+    """Return all skill IDs listed in docs/skills/README.md."""
+    return set(SKILL_PATTERN.findall(readme.read_text(encoding="utf-8")))
+
+
+@pytest.fixture(scope="session")
+def manifested_skills() -> set[str]:
+    return get_manifested_skills(REPO_ROOT / "skills")
+
+
+@pytest.fixture(scope="session")
+def cataloged_skills() -> set[str]:
+    return get_cataloged_skills(REPO_ROOT / "docs" / "skills" / "README.md")
+
+
+@pytest.fixture(scope="session")
+def example_scripts() -> set[str]:
+    examples_dir = REPO_ROOT / "examples"
+    return {
+        path.name
+        for path in examples_dir.glob("*.py")
+        if not path.name.endswith("_common.py")
+        and path.name not in GRANDFATHERED_EXAMPLES
+    }
+
+
+@pytest.fixture(scope="session")
+def indexed_example_scripts() -> set[str]:
+    readme = (REPO_ROOT / "examples" / "README.md").read_text(encoding="utf-8")
+    return set(EXAMPLE_PATTERN.findall(readme))
+
+
+@pytest.fixture(scope="session")
+def agent_loops_text() -> str:
+    return (
+        (REPO_ROOT / "docs" / "usage" / "agent_loops.md")
+        .read_text(encoding="utf-8")
+        .lower()
+    )
+
+
+def test_readme_matches_manifests(
+    cataloged_skills: set[str],
+    manifested_skills: set[str],
+):
+    """README skill index matches manifested skills."""
+
+    missing = manifested_skills - cataloged_skills
+    extra = cataloged_skills - manifested_skills
+
+    assert (
+        not missing
+    ), "Skills with manifest but missing from docs/skills/README.md:\n" + "\n".join(
+        f"  - {skill}" for skill in sorted(missing)
+    )
+
+    assert (
+        not extra
+    ), "Skills listed in docs/skills/README.md without a manifest.yaml:\n" + "\n".join(
+        f"  - {skill}" for skill in sorted(extra)
+    )
+
+
+def test_manifested_skills_have_catalog_pages(
+    manifested_skills: set[str],
+):
+    """Every manifested skill has a catalog page."""
+
+    docs_root = REPO_ROOT / "docs" / "skills"
+
+    missing = [
+        skill
+        for skill in sorted(manifested_skills)
+        if not (docs_root / f"{Path(skill).name}.md").exists()
+    ]
+
+    assert not missing, "Missing catalog pages:\n" + "\n".join(
+        f"  - docs/skills/{Path(skill).name}.md ({skill})" for skill in missing
+    )
+
+
+def test_catalog_pages_have_manifests(
+    manifested_skills: set[str],
+):
+    """Every catalog page corresponds to a manifested skill."""
+
+    docs_root = REPO_ROOT / "docs" / "skills"
+
+    expected = {Path(skill).name for skill in manifested_skills}
+
+    actual = {page.stem for page in docs_root.glob("*.md") if page.name != "README.md"}
+
+    orphaned = actual - expected
+
+    assert not orphaned, "Catalog pages without a matching manifest:\n" + "\n".join(
+        f"  - docs/skills/{page}.md" for page in sorted(orphaned)
+    )
+
+
+def test_examples_readme_matches_files(
+    example_scripts: set[str],
+    indexed_example_scripts: set[str],
+):
+    """Examples README matches runnable scripts."""
+
+    missing = example_scripts - indexed_example_scripts
+    orphaned = indexed_example_scripts - example_scripts
+
+    assert (
+        not missing
+    ), "Example scripts missing from examples/README.md:\n" + "\n".join(
+        f"  - {script}" for script in sorted(missing)
+    )
+
+    assert (
+        not orphaned
+    ), "README references non-existent example scripts:\n" + "\n".join(
+        f"  - {script}" for script in sorted(orphaned)
+    )
+
+
+def test_agent_loops_reference_all_skills(
+    manifested_skills: set[str],
+    agent_loops_text: str,
+):
+    """Every manifested skill is referenced in agent_loops.md."""
+
+    missing = []
+
+    for skill in sorted(manifested_skills):
+        skill_name = Path(skill).name
+
+        if (
+            skill.lower() not in agent_loops_text
+            and skill_name.lower() not in agent_loops_text
+        ):
+            missing.append(skill)
+
+    assert not missing, "Skills missing from docs/usage/agent_loops.md:\n" + "\n".join(
+        f"  - {skill}" for skill in missing
+    )