diff --git a/.github/workflows/_build-docs.yml b/.github/workflows/_build-docs.yml index 81a006f7e..a07cd982e 100644 --- a/.github/workflows/_build-docs.yml +++ b/.github/workflows/_build-docs.yml @@ -59,6 +59,7 @@ jobs: set -euo pipefail mise run docs:generate mise run docs:generate:markdown -- "${TAG}" + mise run docs:generate:json -- "${TAG}" - name: Package documentation env: diff --git a/tasks/docs/generate/json.sh b/tasks/docs/generate/json.sh new file mode 100755 index 000000000..cb0991840 --- /dev/null +++ b/tasks/docs/generate/json.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#MISE description="Generate JSON manifest from XML documentation" +#USAGE arg "version" help="Version to include in the manifest" default="DEV" + +VERSION=${ARGC_VERSION:-DEV} + +echo "Converting XML to JSON manifest..." + +# Ensure XML exists +if [ ! -d "docs/api/xml" ]; then + echo "warning: XML documentation not found" + echo "Generating XML documentation..." + mise run --output prefix docs:generate +fi + +# Emit the authoritative domain/variant matrix straight from the Rust catalog +# (eql_domains::CATALOG) — the source of truth the generated SQL is rendered from. +mkdir -p docs/api/json +echo "Dumping domain catalog (eql-codegen dump-catalog)..." +cargo run -q -p eql-codegen dump-catalog > docs/api/json/eql-catalog.json + +# Run converter (functions from XML, domains from the catalog dump) +mise run --output prefix docs:generate:xml-to-json docs/api/xml docs/api/json "$VERSION" docs/api/json/eql-catalog.json + +echo "" +echo "✓ JSON manifest: docs/api/json/eql-manifest.json" diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py new file mode 100755 index 000000000..0070ac1e1 --- /dev/null +++ b/tasks/docs/generate/test_xml_to_json.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Tests for xml-to-json.py + +Verifies the JSON manifest is built from the same Doxygen XML extraction as +the Markdown reference (reuse of process_function), and has the expected shape. +""" + +import importlib.util +import json +import tempfile +from pathlib import Path + +# xml-to-json.py is hyphenated → load by path. +_spec = importlib.util.spec_from_file_location( + "eql_xml_to_json", Path(__file__).parent / "xml-to-json.py" +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +build_manifest = _mod.build_manifest +load_domains = _mod.load_domains + +# Shape of `eql-codegen dump-catalog` output. +CATALOG_JSON = """{ + "types": [ + { "token": "text", "is_eq_only": false, "domains": [ + { "segment": "storage", "suffix": "", "supported_ops": [] }, + { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"] }, + { "segment": "search", "suffix": "_search", + "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"] } + ]} + ] +}""" + +SAMPLE_XML = """ + + + + hmac_256 + (val jsonb) RETURNS text + + val + jsonb + + Compute the HMAC-SHA-256 term for a value. + + Used for equality search. + + + val + jsonb the encrypted value + + + the HMAC term + + + + + +""" + + +def test_build_manifest_shape(): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "hmac.xml").write_text(SAMPLE_XML) + (Path(d) / "empty-catalog.json").write_text('{"types": []}') + manifest = build_manifest( + Path(d), "1.2.3", catalog_path=Path(d) / "empty-catalog.json" + ) + + assert manifest["name"] == "eql" + assert manifest["version"] == "1.2.3" + assert manifest["counts"]["functions"] == 1 + assert manifest["counts"]["public"] == 1 + + fn = manifest["functions"][0] + assert fn["name"] == "hmac_256" + assert fn["visibility"] == "public" + assert "HMAC" in fn["brief"] + assert fn["returns"]["description"] == "the HMAC term" + assert fn["source"] == {"file": "src/hmac_256/functions.sql", "line": 12} + assert any(p["name"] == "val" for p in fn["params"]) + + # Must be JSON-serializable. + json.dumps(manifest) + + +def test_skips_index_and_doxyfile(): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "index.xml").write_text(SAMPLE_XML) + (Path(d) / "Doxyfile.xml").write_text(SAMPLE_XML) + manifest = build_manifest(Path(d), "DEV") + assert manifest["counts"]["functions"] == 0 + + +def test_load_domains(): + with tempfile.TemporaryDirectory() as d: + cat = Path(d) / "eql-catalog.json" + cat.write_text(CATALOG_JSON) + domains = load_domains(cat) + + by_name = {x["name"]: x for x in domains} + assert by_name["public.text"]["capabilities"] == ["storage"] + assert by_name["public.text_eq"]["type"] == "text" + assert by_name["public.text_eq"]["variant"] == "eq" + assert by_name["public.text_eq"]["capabilities"] == ["equality"] + assert by_name["public.text_eq"]["supportedOperators"] == ["=", "<>"] + # text_search carries all three capabilities, derived from its operators. + assert by_name["public.text_search"]["capabilities"] == ["equality", "order", "match"] + + +if __name__ == "__main__": + test_build_manifest_shape() + test_skips_index_and_doxyfile() + test_load_domains() + print("✓ all tests passed") diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py new file mode 100755 index 000000000..25ec2eeac --- /dev/null +++ b/tasks/docs/generate/xml-to-json.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +#MISE hide=true +""" +Doxygen XML -> structured JSON manifest for EQL's SQL API. + +A machine-readable companion to xml-to-markdown.py. Same Doxygen XML, same +SQL-via-Doxygen quirk handling (parameter name/type swap, operator names in +brief text, RETURNS extraction) — this emits JSON instead of Markdown, for +downstream consumers: docs generation, agents, and drift checks against the +hand-written reference. + +Reuses the extraction in xml-to-markdown.py so the manifest and the Markdown +reference can never diverge in how they read the XML. + +Doxygen does not extract CREATE DOMAIN, so the manifest reads the encrypted +domain/variant matrix (the core of the EQL v3 surface) straight from the Rust +catalog via `eql-codegen dump-catalog` — authoritative type names + operators. + +Usage: xml-to-json.py [output_dir] [version] [catalog_json] +""" + +import importlib.util +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +# xml-to-markdown.py has a hyphen in its name, so it can't be `import`ed +# normally; load it by path and reuse process_function verbatim. +_spec = importlib.util.spec_from_file_location( + "eql_xml_to_markdown", Path(__file__).parent / "xml-to-markdown.py" +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +process_function = _mod.process_function + + +def _strip_ticks(value): + return value.strip("`").strip() if value else "" + + +def _int_or_none(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _to_entry(func): + """Shape one extracted function into a manifest entry.""" + return { + "name": func["name"], + "signature": func["signature"], + "visibility": "private" if func["is_private"] else "public", + "brief": func["brief"], + "description": func["detailed"], + "params": func["params"], # [{name, type, description}] + "returns": { + "type": _strip_ticks(func["return_type"]), + "description": func["return_desc"], + }, + "throws": func["exceptions"], + "notes": func["notes"], + "warnings": func["warnings"], + "seeAlso": func["see_also"], + "source": {"file": func["source"], "line": _int_or_none(func["line"])}, + } + + +# ── Encrypted domains (from the Rust catalog) ──────────────────────────────── +# The domain/variant matrix is the core of the EQL v3 surface. Rather than parse +# the generated SQL (one step removed, format-fragile), read it straight from the +# source of truth: `eql-codegen dump-catalog` serializes eql_domains::CATALOG — +# authoritative type tokens plus the exact SQL operators each domain supports. +def _capabilities_from_ops(ops): + caps = [] + if any(o in ops for o in ("=", "<>")): + caps.append("equality") + if any(o in ops for o in ("<", "<=", ">", ">=")): + caps.append("order") + if any(o in ops for o in ("@>", "<@")): + caps.append("match") + return caps or ["storage"] + + +def load_domains(catalog_path: Path) -> list: + """Map `eql-codegen dump-catalog` JSON into manifest domain entries.""" + if not catalog_path.exists(): + print(f"Warning: catalog dump not found: {catalog_path}; skipping domains", file=sys.stderr) + return [] + + catalog = json.loads(catalog_path.read_text()) + domains = [] + for type_entry in catalog.get("types", []): + token = type_entry["token"] + for dom in type_entry["domains"]: + suffix = dom.get("suffix", "") + ops = dom.get("supported_ops", []) + domains.append({ + # v3 user domains live in the `public` schema (public-domain + # migration on eql_v3); the catalog dump emits the bare token. + "name": f"public.{token}{suffix}", + "type": token, + "variant": suffix.lstrip("_"), + "base": "jsonb", + "capabilities": _capabilities_from_ops(ops), + "supportedOperators": ops, + }) + domains.sort(key=lambda d: (d["type"], d["name"])) + return domains + + +def build_manifest(xml_dir: Path, version: str, catalog_path: Path = Path("docs/api/json/eql-catalog.json")) -> dict: + functions = [] + for xml_file in sorted(xml_dir.glob("*.xml")): + if xml_file.name in ("index.xml", "Doxyfile.xml"): + continue + try: + root = ET.parse(xml_file).getroot() + except ET.ParseError as exc: + print(f"Warning: failed to parse {xml_file.name}: {exc}", file=sys.stderr) + continue + for memberdef in root.findall('.//memberdef[@kind="function"]'): + func = process_function(memberdef) + if func: + functions.append(func) + + functions.sort(key=lambda f: (f["is_private"], f["name"], f["signature"])) + domains = load_domains(catalog_path) + + return { + "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json", + "name": "eql", + "version": version, + "generatedFrom": "doxygen-xml + catalog", + "counts": { + "functions": len(functions), + "public": sum(1 for f in functions if not f["is_private"]), + "private": sum(1 for f in functions if f["is_private"]), + "domains": len(domains), + }, + "functions": [_to_entry(f) for f in functions], + "domains": domains, + } + + +def main(): + if len(sys.argv) < 2: + print("Usage: xml-to-json.py [output_dir] [version] [catalog_json]") + sys.exit(1) + + xml_dir = Path(sys.argv[1]) + output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("docs/api/json") + version = sys.argv[3] if len(sys.argv) > 3 else "DEV" + catalog_path = ( + Path(sys.argv[4]) if len(sys.argv) > 4 else Path("docs/api/json/eql-catalog.json") + ) + + if not xml_dir.exists(): + print(f"Error: XML directory not found: {xml_dir}") + sys.exit(1) + + manifest = build_manifest(xml_dir, version, catalog_path) + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "eql-manifest.json" + output_file.write_text(json.dumps(manifest, indent=2) + "\n") + + counts = manifest["counts"] + print(f"✓ Generated JSON manifest: {output_file}") + print( + f" Functions: {counts['functions']} " + f"({counts['public']} public, {counts['private']} private)" + ) + print(f" Domains: {counts['domains']}") + + +if __name__ == "__main__": + main() diff --git a/tasks/docs/package.sh b/tasks/docs/package.sh index 81ed50e0c..89706b3d7 100755 --- a/tasks/docs/package.sh +++ b/tasks/docs/package.sh @@ -28,6 +28,12 @@ if [ ! -d "${DOCS_DIR}/xml" ] || [ -z "$(ls -A ${DOCS_DIR}/xml/*.xml 2>/dev/null exit 1 fi +if [ ! -f "${DOCS_DIR}/json/eql-manifest.json" ]; then + echo "Error: ${DOCS_DIR}/json/eql-manifest.json not found" + echo "Run 'mise run docs:generate:json' first to generate the JSON manifest" + exit 1 +fi + # Create output directory @@ -38,11 +44,11 @@ echo "Creating archives..." cd "${DOCS_DIR}" # Create ZIP archive with all documentation formats -zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md xml/*.xml html/ +zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md json/eql-manifest.json xml/*.xml html/ echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.zip" # Create tarball with all documentation formats -tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md xml/ html/ +tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md json/ xml/ html/ echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" cd ../..