docs: emit a JSON API manifest from the Doxygen XML (EQL v3)

.github/workflows/_build-docs.yml

@@ -59,6 +59,7 @@ jobs:
           set -euo pipefail
           mise run docs:generate
           mise run docs:generate:markdown -- "${TAG}"
+          mise run docs:generate:json -- "${TAG}"
 
       - name: Package documentation
         env:

tasks/docs/generate/json.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#MISE description="Generate JSON manifest from XML documentation"
+#USAGE arg "version" help="Version to include in the manifest" default="DEV"
+
+VERSION=${ARGC_VERSION:-DEV}
+
+echo "Converting XML to JSON manifest..."
+
+# Ensure XML exists
+if [ ! -d "docs/api/xml" ]; then
+  echo "warning: XML documentation not found"
+  echo "Generating XML documentation..."
+  mise run --output prefix docs:generate
+fi
+
+# Emit the authoritative domain/variant matrix straight from the Rust catalog
+# (eql_domains::CATALOG) — the source of truth the generated SQL is rendered from.
+mkdir -p docs/api/json
+echo "Dumping domain catalog (eql-codegen dump-catalog)..."
+cargo run -q -p eql-codegen dump-catalog > docs/api/json/eql-catalog.json
+
+# Run converter (functions from XML, domains from the catalog dump)
+mise run --output prefix docs:generate:xml-to-json docs/api/xml docs/api/json "$VERSION" docs/api/json/eql-catalog.json
+
+echo ""
+echo "✓ JSON manifest: docs/api/json/eql-manifest.json"

tasks/docs/generate/test_xml_to_json.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Tests for xml-to-json.py
+
+Verifies the JSON manifest is built from the same Doxygen XML extraction as
+the Markdown reference (reuse of process_function), and has the expected shape.
+"""
+
+import importlib.util
+import json
+import tempfile
+from pathlib import Path
+
+# xml-to-json.py is hyphenated → load by path.
+_spec = importlib.util.spec_from_file_location(
+    "eql_xml_to_json", Path(__file__).parent / "xml-to-json.py"
+)
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+build_manifest = _mod.build_manifest
+load_domains = _mod.load_domains
+
+# Shape of `eql-codegen dump-catalog` output.
+CATALOG_JSON = """{
+  "types": [
+    { "token": "text", "is_eq_only": false, "domains": [
+      { "segment": "storage", "suffix": "", "supported_ops": [] },
+      { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"] },
+      { "segment": "search", "suffix": "_search",
+        "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"] }
+    ]}
+  ]
+}"""
+
+SAMPLE_XML = """<?xml version="1.0"?>
+<doxygen>
+  <compounddef>
+    <memberdef kind="function">
+      <name>hmac_256</name>
+      <argsstring>(val jsonb) RETURNS text</argsstring>
+      <param>
+        <type><ref>val</ref> </type>
+        <declname>jsonb</declname>
+      </param>
+      <briefdescription><para>Compute the HMAC-SHA-256 term for a value.</para></briefdescription>
+      <detaileddescription>
+        <para>Used for equality search.
+          <parameterlist kind="param">
+            <parameteritem>
+              <parameternamelist><parametername>val</parametername></parameternamelist>
+              <parameterdescription><para>jsonb the encrypted value</para></parameterdescription>
+            </parameteritem>
+          </parameterlist>
+          <simplesect kind="return"><para>the HMAC term</para></simplesect>
+        </para>
+      </detaileddescription>
+      <location file="src/hmac_256/functions.sql" line="12"/>
+    </memberdef>
+  </compounddef>
+</doxygen>"""
+
+
+def test_build_manifest_shape():
+    with tempfile.TemporaryDirectory() as d:
+        (Path(d) / "hmac.xml").write_text(SAMPLE_XML)
+        (Path(d) / "empty-catalog.json").write_text('{"types": []}')
+        manifest = build_manifest(
+            Path(d), "1.2.3", catalog_path=Path(d) / "empty-catalog.json"
+        )
+
+    assert manifest["name"] == "eql"
+    assert manifest["version"] == "1.2.3"
+    assert manifest["counts"]["functions"] == 1
+    assert manifest["counts"]["public"] == 1
+
+    fn = manifest["functions"][0]
+    assert fn["name"] == "hmac_256"
+    assert fn["visibility"] == "public"
+    assert "HMAC" in fn["brief"]
+    assert fn["returns"]["description"] == "the HMAC term"
+    assert fn["source"] == {"file": "src/hmac_256/functions.sql", "line": 12}
+    assert any(p["name"] == "val" for p in fn["params"])
+
+    # Must be JSON-serializable.
+    json.dumps(manifest)
+
+
+def test_skips_index_and_doxyfile():
+    with tempfile.TemporaryDirectory() as d:
+        (Path(d) / "index.xml").write_text(SAMPLE_XML)
+        (Path(d) / "Doxyfile.xml").write_text(SAMPLE_XML)
+        manifest = build_manifest(Path(d), "DEV")
+    assert manifest["counts"]["functions"] == 0
+
+
+def test_load_domains():
+    with tempfile.TemporaryDirectory() as d:
+        cat = Path(d) / "eql-catalog.json"
+        cat.write_text(CATALOG_JSON)
+        domains = load_domains(cat)
+
+    by_name = {x["name"]: x for x in domains}
+    assert by_name["public.text"]["capabilities"] == ["storage"]
+    assert by_name["public.text_eq"]["type"] == "text"
+    assert by_name["public.text_eq"]["variant"] == "eq"
+    assert by_name["public.text_eq"]["capabilities"] == ["equality"]
+    assert by_name["public.text_eq"]["supportedOperators"] == ["=", "<>"]
+    # text_search carries all three capabilities, derived from its operators.
+    assert by_name["public.text_search"]["capabilities"] == ["equality", "order", "match"]
+
+
+if __name__ == "__main__":
+    test_build_manifest_shape()
+    test_skips_index_and_doxyfile()
+    test_load_domains()
+    print("✓ all tests passed")

tasks/docs/generate/xml-to-json.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+#MISE hide=true
+"""
+Doxygen XML -> structured JSON manifest for EQL's SQL API.
+
+A machine-readable companion to xml-to-markdown.py. Same Doxygen XML, same
+SQL-via-Doxygen quirk handling (parameter name/type swap, operator names in
+brief text, RETURNS extraction) — this emits JSON instead of Markdown, for
+downstream consumers: docs generation, agents, and drift checks against the
+hand-written reference.
+
+Reuses the extraction in xml-to-markdown.py so the manifest and the Markdown
+reference can never diverge in how they read the XML.
+
+Doxygen does not extract CREATE DOMAIN, so the manifest reads the encrypted
+domain/variant matrix (the core of the EQL v3 surface) straight from the Rust
+catalog via `eql-codegen dump-catalog` — authoritative type names + operators.
+
+Usage: xml-to-json.py <xml_dir> [output_dir] [version] [catalog_json]
+"""
+
+import importlib.util
+import json
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+# xml-to-markdown.py has a hyphen in its name, so it can't be `import`ed
+# normally; load it by path and reuse process_function verbatim.
+_spec = importlib.util.spec_from_file_location(
+    "eql_xml_to_markdown", Path(__file__).parent / "xml-to-markdown.py"
+)
+_mod = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_mod)
+process_function = _mod.process_function
+
+
+def _strip_ticks(value):
+    return value.strip("`").strip() if value else ""
+
+
+def _int_or_none(value):
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _to_entry(func):
+    """Shape one extracted function into a manifest entry."""
+    return {
+        "name": func["name"],
+        "signature": func["signature"],
+        "visibility": "private" if func["is_private"] else "public",
+        "brief": func["brief"],
+        "description": func["detailed"],
+        "params": func["params"],  # [{name, type, description}]
+        "returns": {
+            "type": _strip_ticks(func["return_type"]),
+            "description": func["return_desc"],
+        },
+        "throws": func["exceptions"],
+        "notes": func["notes"],
+        "warnings": func["warnings"],
+        "seeAlso": func["see_also"],
+        "source": {"file": func["source"], "line": _int_or_none(func["line"])},
+    }
+
+
+# ── Encrypted domains (from the Rust catalog) ────────────────────────────────
+# The domain/variant matrix is the core of the EQL v3 surface. Rather than parse
+# the generated SQL (one step removed, format-fragile), read it straight from the
+# source of truth: `eql-codegen dump-catalog` serializes eql_domains::CATALOG —
+# authoritative type tokens plus the exact SQL operators each domain supports.
+def _capabilities_from_ops(ops):
+    caps = []
+    if any(o in ops for o in ("=", "<>")):
+        caps.append("equality")
+    if any(o in ops for o in ("<", "<=", ">", ">=")):
+        caps.append("order")
+    if any(o in ops for o in ("@>", "<@")):
+        caps.append("match")
+    return caps or ["storage"]
+
+
+def load_domains(catalog_path: Path) -> list:
+    """Map `eql-codegen dump-catalog` JSON into manifest domain entries."""
+    if not catalog_path.exists():
+        print(f"Warning: catalog dump not found: {catalog_path}; skipping domains", file=sys.stderr)
+        return []
+
+    catalog = json.loads(catalog_path.read_text())
+    domains = []
+    for type_entry in catalog.get("types", []):
+        token = type_entry["token"]
+        for dom in type_entry["domains"]:
+            suffix = dom.get("suffix", "")
+            ops = dom.get("supported_ops", [])
+            domains.append({
+                # v3 user domains live in the `public` schema (public-domain
+                # migration on eql_v3); the catalog dump emits the bare token.
+                "name": f"public.{token}{suffix}",
+                "type": token,
+                "variant": suffix.lstrip("_"),
+                "base": "jsonb",
+                "capabilities": _capabilities_from_ops(ops),
+                "supportedOperators": ops,
+            })
+    domains.sort(key=lambda d: (d["type"], d["name"]))
+    return domains
+
+
+def build_manifest(xml_dir: Path, version: str, catalog_path: Path = Path("docs/api/json/eql-catalog.json")) -> dict:
+    functions = []
+    for xml_file in sorted(xml_dir.glob("*.xml")):
+        if xml_file.name in ("index.xml", "Doxyfile.xml"):
+            continue
+        try:
+            root = ET.parse(xml_file).getroot()
+        except ET.ParseError as exc:
+            print(f"Warning: failed to parse {xml_file.name}: {exc}", file=sys.stderr)
+            continue
+        for memberdef in root.findall('.//memberdef[@kind="function"]'):
+            func = process_function(memberdef)
+            if func:
+                functions.append(func)
+
+    functions.sort(key=lambda f: (f["is_private"], f["name"], f["signature"]))
+    domains = load_domains(catalog_path)
+
+    return {
+        "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json",
+        "name": "eql",
+        "version": version,
+        "generatedFrom": "doxygen-xml + catalog",
+        "counts": {
+            "functions": len(functions),
+            "public": sum(1 for f in functions if not f["is_private"]),
+            "private": sum(1 for f in functions if f["is_private"]),
+            "domains": len(domains),
+        },
+        "functions": [_to_entry(f) for f in functions],
+        "domains": domains,
+    }
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: xml-to-json.py <xml_dir> [output_dir] [version] [catalog_json]")
+        sys.exit(1)
+
+    xml_dir = Path(sys.argv[1])
+    output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("docs/api/json")
+    version = sys.argv[3] if len(sys.argv) > 3 else "DEV"
+    catalog_path = (
+        Path(sys.argv[4]) if len(sys.argv) > 4 else Path("docs/api/json/eql-catalog.json")
+    )
+
+    if not xml_dir.exists():
+        print(f"Error: XML directory not found: {xml_dir}")
+        sys.exit(1)
+
+    manifest = build_manifest(xml_dir, version, catalog_path)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_file = output_dir / "eql-manifest.json"
+    output_file.write_text(json.dumps(manifest, indent=2) + "\n")
+
+    counts = manifest["counts"]
+    print(f"✓ Generated JSON manifest: {output_file}")
+    print(
+        f"  Functions: {counts['functions']} "
+        f"({counts['public']} public, {counts['private']} private)"
+    )
+    print(f"  Domains: {counts['domains']}")
+
+
+if __name__ == "__main__":
+    main()

tasks/docs/package.sh

@@ -28,6 +28,12 @@ if [ ! -d "${DOCS_DIR}/xml" ] || [ -z "$(ls -A ${DOCS_DIR}/xml/*.xml 2>/dev/null
   exit 1
 fi
 
+if [ ! -f "${DOCS_DIR}/json/eql-manifest.json" ]; then
+  echo "Error: ${DOCS_DIR}/json/eql-manifest.json not found"
+  echo "Run 'mise run docs:generate:json' first to generate the JSON manifest"
+  exit 1
+fi
+
 
 
 # Create output directory
@@ -38,11 +44,11 @@ echo "Creating archives..."
 cd "${DOCS_DIR}"
 
 # Create ZIP archive with all documentation formats
-zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md xml/*.xml html/
+zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md json/eql-manifest.json xml/*.xml html/
 echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.zip"
 
 # Create tarball with all documentation formats
-tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md xml/ html/
+tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md json/ xml/ html/
 echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz"
 
 cd ../..