From 09297df3136c48cab628f56c79e766a1c246739d Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 6 Jul 2026 00:29:31 +1000 Subject: [PATCH 1/3] docs: emit a JSON API manifest alongside the Markdown reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `docs:generate:json` (tasks/docs/generate/xml-to-json.py), which converts the Doxygen XML into a structured `eql-manifest.json`, reusing xml-to-markdown's extraction so the JSON manifest and the Markdown reference never diverge in how they read the XML. Packaged into the `eql-docs-` release archives and run in the release workflow after the Markdown step. The manifest is a machine-readable API surface (per function: signature, brief, description, params, returns, throws, notes, source) for downstream consumers — docs generation, agents, and drift-checking hand-written reference pages against the shipped EQL version. Additive; the Markdown path and outputs are unchanged. Includes tasks/docs/generate/test_xml_to_json.py. Claude-Session: https://claude.ai/code/session_01CqDNqLSEEkCi7xAJFq7HJA --- .github/workflows/_build-docs.yml | 1 + tasks/docs/generate/json.sh | 20 ++++ tasks/docs/generate/test_xml_to_json.py | 83 ++++++++++++++++ tasks/docs/generate/xml-to-json.py | 124 ++++++++++++++++++++++++ tasks/docs/package.sh | 10 +- 5 files changed, 236 insertions(+), 2 deletions(-) create mode 100755 tasks/docs/generate/json.sh create mode 100755 tasks/docs/generate/test_xml_to_json.py create mode 100755 tasks/docs/generate/xml-to-json.py diff --git a/.github/workflows/_build-docs.yml b/.github/workflows/_build-docs.yml index 81a006f7e..a07cd982e 100644 --- a/.github/workflows/_build-docs.yml +++ b/.github/workflows/_build-docs.yml @@ -59,6 +59,7 @@ jobs: set -euo pipefail mise run docs:generate mise run docs:generate:markdown -- "${TAG}" + mise run docs:generate:json -- "${TAG}" - name: Package documentation env: diff --git a/tasks/docs/generate/json.sh b/tasks/docs/generate/json.sh new file mode 100755 index 000000000..91316701e --- /dev/null +++ b/tasks/docs/generate/json.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +#MISE description="Generate JSON manifest from XML documentation" +#USAGE arg "version" help="Version to include in the manifest" default="DEV" + +VERSION=${ARGC_VERSION:-DEV} + +echo "Converting XML to JSON manifest..." + +# Ensure XML exists +if [ ! -d "docs/api/xml" ]; then + echo "warning: XML documentation not found" + echo "Generating XML documentation..." + mise run --output prefix docs:generate +fi + +# Run converter +mise run --output prefix docs:generate:xml-to-json docs/api/xml docs/api/json "$VERSION" + +echo "" +echo "✓ JSON manifest: docs/api/json/eql-manifest.json" diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py new file mode 100755 index 000000000..c7c7ca583 --- /dev/null +++ b/tasks/docs/generate/test_xml_to_json.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +Tests for xml-to-json.py + +Verifies the JSON manifest is built from the same Doxygen XML extraction as +the Markdown reference (reuse of process_function), and has the expected shape. +""" + +import importlib.util +import json +import tempfile +from pathlib import Path + +# xml-to-json.py is hyphenated → load by path. +_spec = importlib.util.spec_from_file_location( + "eql_xml_to_json", Path(__file__).parent / "xml-to-json.py" +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +build_manifest = _mod.build_manifest + +SAMPLE_XML = """ + + + + hmac_256 + (val jsonb) RETURNS text + + val + jsonb + + Compute the HMAC-SHA-256 term for a value. + + Used for equality search. + + + val + jsonb the encrypted value + + + the HMAC term + + + + + +""" + + +def test_build_manifest_shape(): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "hmac.xml").write_text(SAMPLE_XML) + manifest = build_manifest(Path(d), "1.2.3") + + assert manifest["name"] == "eql" + assert manifest["version"] == "1.2.3" + assert manifest["counts"]["functions"] == 1 + assert manifest["counts"]["public"] == 1 + + fn = manifest["functions"][0] + assert fn["name"] == "hmac_256" + assert fn["visibility"] == "public" + assert "HMAC" in fn["brief"] + assert fn["returns"]["description"] == "the HMAC term" + assert fn["source"] == {"file": "src/hmac_256/functions.sql", "line": 12} + assert any(p["name"] == "val" for p in fn["params"]) + + # Must be JSON-serializable. + json.dumps(manifest) + + +def test_skips_index_and_doxyfile(): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "index.xml").write_text(SAMPLE_XML) + (Path(d) / "Doxyfile.xml").write_text(SAMPLE_XML) + manifest = build_manifest(Path(d), "DEV") + assert manifest["counts"]["functions"] == 0 + + +if __name__ == "__main__": + test_build_manifest_shape() + test_skips_index_and_doxyfile() + print("✓ all tests passed") diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py new file mode 100755 index 000000000..d75b72c5c --- /dev/null +++ b/tasks/docs/generate/xml-to-json.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +#MISE hide=true +""" +Doxygen XML -> structured JSON manifest for EQL's SQL API. + +A machine-readable companion to xml-to-markdown.py. Same Doxygen XML, same +SQL-via-Doxygen quirk handling (parameter name/type swap, operator names in +brief text, RETURNS extraction) — this emits JSON instead of Markdown, for +downstream consumers: docs generation, agents, and drift checks against the +hand-written reference. + +Reuses the extraction in xml-to-markdown.py so the manifest and the Markdown +reference can never diverge in how they read the XML. + +Usage: xml-to-json.py [output_dir] [version] +""" + +import importlib.util +import json +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +# xml-to-markdown.py has a hyphen in its name, so it can't be `import`ed +# normally; load it by path and reuse process_function verbatim. +_spec = importlib.util.spec_from_file_location( + "eql_xml_to_markdown", Path(__file__).parent / "xml-to-markdown.py" +) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +process_function = _mod.process_function + + +def _strip_ticks(value): + return value.strip("`").strip() if value else "" + + +def _int_or_none(value): + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _to_entry(func): + """Shape one extracted function into a manifest entry.""" + return { + "name": func["name"], + "signature": func["signature"], + "visibility": "private" if func["is_private"] else "public", + "brief": func["brief"], + "description": func["detailed"], + "params": func["params"], # [{name, type, description}] + "returns": { + "type": _strip_ticks(func["return_type"]), + "description": func["return_desc"], + }, + "throws": func["exceptions"], + "notes": func["notes"], + "warnings": func["warnings"], + "seeAlso": func["see_also"], + "source": {"file": func["source"], "line": _int_or_none(func["line"])}, + } + + +def build_manifest(xml_dir: Path, version: str) -> dict: + functions = [] + for xml_file in sorted(xml_dir.glob("*.xml")): + if xml_file.name in ("index.xml", "Doxyfile.xml"): + continue + try: + root = ET.parse(xml_file).getroot() + except ET.ParseError as exc: + print(f"Warning: failed to parse {xml_file.name}: {exc}", file=sys.stderr) + continue + for memberdef in root.findall('.//memberdef[@kind="function"]'): + func = process_function(memberdef) + if func: + functions.append(func) + + functions.sort(key=lambda f: (f["is_private"], f["name"], f["signature"])) + + return { + "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json", + "name": "eql", + "version": version, + "generatedFrom": "doxygen-xml", + "counts": { + "functions": len(functions), + "public": sum(1 for f in functions if not f["is_private"]), + "private": sum(1 for f in functions if f["is_private"]), + }, + "functions": [_to_entry(f) for f in functions], + } + + +def main(): + if len(sys.argv) < 2: + print("Usage: xml-to-json.py [output_dir] [version]") + sys.exit(1) + + xml_dir = Path(sys.argv[1]) + output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("docs/api/json") + version = sys.argv[3] if len(sys.argv) > 3 else "DEV" + + if not xml_dir.exists(): + print(f"Error: XML directory not found: {xml_dir}") + sys.exit(1) + + manifest = build_manifest(xml_dir, version) + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "eql-manifest.json" + output_file.write_text(json.dumps(manifest, indent=2) + "\n") + + counts = manifest["counts"] + print(f"✓ Generated JSON manifest: {output_file}") + print( + f" Functions: {counts['functions']} " + f"({counts['public']} public, {counts['private']} private)" + ) + + +if __name__ == "__main__": + main() diff --git a/tasks/docs/package.sh b/tasks/docs/package.sh index 81ed50e0c..89706b3d7 100755 --- a/tasks/docs/package.sh +++ b/tasks/docs/package.sh @@ -28,6 +28,12 @@ if [ ! -d "${DOCS_DIR}/xml" ] || [ -z "$(ls -A ${DOCS_DIR}/xml/*.xml 2>/dev/null exit 1 fi +if [ ! -f "${DOCS_DIR}/json/eql-manifest.json" ]; then + echo "Error: ${DOCS_DIR}/json/eql-manifest.json not found" + echo "Run 'mise run docs:generate:json' first to generate the JSON manifest" + exit 1 +fi + # Create output directory @@ -38,11 +44,11 @@ echo "Creating archives..." cd "${DOCS_DIR}" # Create ZIP archive with all documentation formats -zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md xml/*.xml html/ +zip -r -q "../../${OUTPUT_DIR}/eql-docs-${VERSION}.zip" markdown/API.md json/eql-manifest.json xml/*.xml html/ echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.zip" # Create tarball with all documentation formats -tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md xml/ html/ +tar czf "../../${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" markdown/API.md json/ xml/ html/ echo "Created ${OUTPUT_DIR}/eql-docs-${VERSION}.tar.gz" cd ../.. From 8bd430a2e8b3bbe28284f2189c3670501ee8d0cf Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 6 Jul 2026 09:54:11 +1000 Subject: [PATCH 2/3] docs: include the encrypted domain/variant matrix in the manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doxygen doesn't extract CREATE DOMAIN, but the domain variants (eql_v3._eq / _ord / _ord_ore / _ord_ope / _match / _search) are the core of the EQL v3 surface — arguably the most important thing to document. Parse the generated `src/v3/**/*_types.sql` (source of truth: the Rust catalog in crates/eql-domains) and derive each domain's capability structurally from its CHECK keys — hm=equality, ob/op=order, bf=match, sv=json — plus the extractor function per term (hmac_256 / ore_block_256 / bloom_filter). The manifest now carries a `domains` array (name, type, variant, base, terms, capabilities, termFunctions, source) alongside `functions`. No DB or Rust build required. Verified against the real v3 SQL (51 domains): e.g. text_search → [equality, order, match], integer_ord → [order], storage-only types → [storage]. Claude-Session: https://claude.ai/code/session_01CqDNqLSEEkCi7xAJFq7HJA --- tasks/docs/generate/test_xml_to_json.py | 36 +++++++- tasks/docs/generate/xml-to-json.py | 109 ++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 6 deletions(-) diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py index c7c7ca583..8798d20c8 100755 --- a/tasks/docs/generate/test_xml_to_json.py +++ b/tasks/docs/generate/test_xml_to_json.py @@ -18,6 +18,23 @@ _mod = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_mod) build_manifest = _mod.build_manifest +parse_domains = _mod.parse_domains + +DOMAIN_SQL = """DO $$ +BEGIN + --! @brief Encrypted domain eql_v3.text_eq. + IF NOT EXISTS (SELECT 1) THEN + CREATE DOMAIN eql_v3.text_eq AS jsonb + CHECK ( + jsonb_typeof(VALUE) = 'object' + AND VALUE ? 'v' + AND VALUE ? 'i' + AND VALUE ? 'c' + AND VALUE ? 'hm' + AND VALUE->>'v' = '3' + ); + END IF; +END $$;""" SAMPLE_XML = """ @@ -50,7 +67,7 @@ def test_build_manifest_shape(): with tempfile.TemporaryDirectory() as d: (Path(d) / "hmac.xml").write_text(SAMPLE_XML) - manifest = build_manifest(Path(d), "1.2.3") + manifest = build_manifest(Path(d), "1.2.3", src_dir=Path(d)) assert manifest["name"] == "eql" assert manifest["version"] == "1.2.3" @@ -77,7 +94,24 @@ def test_skips_index_and_doxyfile(): assert manifest["counts"]["functions"] == 0 +def test_parse_domains(): + with tempfile.TemporaryDirectory() as d: + (Path(d) / "text_types.sql").write_text(DOMAIN_SQL) + domains = parse_domains(Path(d)) + + assert len(domains) == 1 + dm = domains[0] + assert dm["name"] == "eql_v3.text_eq" + assert dm["type"] == "text" + assert dm["variant"] == "eq" + assert dm["terms"] == ["hm"] # envelope keys (v/i/c) excluded + assert dm["capabilities"] == ["equality"] + assert dm["termFunctions"] == ["eql_v3.hmac_256"] + assert dm["brief"] == "Encrypted domain eql_v3.text_eq." + + if __name__ == "__main__": test_build_manifest_shape() test_skips_index_and_doxyfile() + test_parse_domains() print("✓ all tests passed") diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py index d75b72c5c..efc569851 100755 --- a/tasks/docs/generate/xml-to-json.py +++ b/tasks/docs/generate/xml-to-json.py @@ -12,11 +12,16 @@ Reuses the extraction in xml-to-markdown.py so the manifest and the Markdown reference can never diverge in how they read the XML. -Usage: xml-to-json.py [output_dir] [version] +Doxygen does not extract CREATE DOMAIN, so the manifest also parses the +generated domain SQL to emit the encrypted domain/variant matrix (the core of +the EQL v3 surface) — each domain's capability derived from its CHECK keys. + +Usage: xml-to-json.py [output_dir] [version] [sql_src_dir] """ import importlib.util import json +import re import sys import xml.etree.ElementTree as ET from pathlib import Path @@ -63,7 +68,96 @@ def _to_entry(func): } -def build_manifest(xml_dir: Path, version: str) -> dict: +# ── Encrypted domains ──────────────────────────────────────────────────────── +# Doxygen does not extract CREATE DOMAIN, but the domain/variant matrix is the +# core of the EQL v3 surface. The generated `*_types.sql` (source of truth: the +# Rust catalog in crates/eql-domains) encodes each domain's capability +# STRUCTURALLY as the required CHECK keys, so we derive it directly: +# hm = HMAC equality · ob = ORE order · op = OPE order · bf = bloom match · +# sv = STE-vec (JSON). v/i/c are the envelope, not index terms. +_ENVELOPE_KEYS = {"v", "i", "c", "k"} +_TERM_CAPABILITY = { + "hm": "equality", + "ob": "order", + "op": "order", + "bf": "match", + "sv": "json", +} +# Term -> extractor function (from crates/eql-domains/src/term.rs). +_TERM_FUNCTION = { + "hm": "eql_v3.hmac_256", + "ob": "eql_v3.ore_block_256", + "bf": "eql_v3.bloom_filter", +} +# Longest-first so `_ord_ore` wins over `_ord`. +_VARIANT_SUFFIXES = ("_ord_ore", "_ord_ope", "_ord", "_eq", "_match", "_search") + +_DOMAIN_RE = re.compile(r"CREATE DOMAIN eql_v3\.([a-z0-9_]+)\s+AS\s+([a-z_]+)", re.I) +_KEY_RE = re.compile(r"VALUE \? '([a-z0-9]+)'") +_BRIEF_RE = re.compile(r"--!\s*@brief\s+(.*)") + + +def _build_domain(name, base, brief, terms, source_file, line): + scalar_type, variant = name, "" + for suffix in _VARIANT_SUFFIXES: + if name.endswith(suffix): + scalar_type, variant = name[: -len(suffix)], suffix[1:] + break + + capabilities = [] + for term in terms: + cap = _TERM_CAPABILITY.get(term) + if cap and cap not in capabilities: + capabilities.append(cap) + if not capabilities: + capabilities = ["storage"] + + return { + "name": f"eql_v3.{name}", + "type": scalar_type, + "variant": variant, + "base": base, + "brief": brief, + "terms": terms, + "capabilities": capabilities, + "termFunctions": [_TERM_FUNCTION[t] for t in terms if t in _TERM_FUNCTION], + "source": {"file": str(source_file), "line": line}, + } + + +def parse_domains(src_dir: Path) -> list: + """Extract eql_v3 CREATE DOMAIN definitions + their capability from the SQL.""" + if not src_dir.exists(): + print(f"Warning: SQL source dir not found: {src_dir}; skipping domains", file=sys.stderr) + return [] + + domains = [] + for sql_file in sorted(src_dir.rglob("*.sql")): + lines = sql_file.read_text().splitlines() + last_brief = "" + for i, line in enumerate(lines): + brief_match = _BRIEF_RE.search(line) + if brief_match: + last_brief = brief_match.group(1).strip() + domain_match = _DOMAIN_RE.search(line) + if not domain_match: + continue + name, base = domain_match.group(1), domain_match.group(2) + # Collect CHECK keys until the block closes or the next domain begins. + keys = [] + for follow in lines[i + 1:]: + if _DOMAIN_RE.search(follow) or re.match(r"\s*\);", follow): + break + keys.extend(_KEY_RE.findall(follow)) + terms = [k for k in dict.fromkeys(keys) if k not in _ENVELOPE_KEYS] + domains.append(_build_domain(name, base, last_brief, terms, sql_file, i + 1)) + last_brief = "" + + domains.sort(key=lambda d: (d["type"], d["name"])) + return domains + + +def build_manifest(xml_dir: Path, version: str, src_dir: Path = Path("src/v3")) -> dict: functions = [] for xml_file in sorted(xml_dir.glob("*.xml")): if xml_file.name in ("index.xml", "Doxyfile.xml"): @@ -79,35 +173,39 @@ def build_manifest(xml_dir: Path, version: str) -> dict: functions.append(func) functions.sort(key=lambda f: (f["is_private"], f["name"], f["signature"])) + domains = parse_domains(src_dir) return { "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json", "name": "eql", "version": version, - "generatedFrom": "doxygen-xml", + "generatedFrom": "doxygen-xml + sql-domains", "counts": { "functions": len(functions), "public": sum(1 for f in functions if not f["is_private"]), "private": sum(1 for f in functions if f["is_private"]), + "domains": len(domains), }, "functions": [_to_entry(f) for f in functions], + "domains": domains, } def main(): if len(sys.argv) < 2: - print("Usage: xml-to-json.py [output_dir] [version]") + print("Usage: xml-to-json.py [output_dir] [version] [sql_src_dir]") sys.exit(1) xml_dir = Path(sys.argv[1]) output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("docs/api/json") version = sys.argv[3] if len(sys.argv) > 3 else "DEV" + src_dir = Path(sys.argv[4]) if len(sys.argv) > 4 else Path("src/v3") if not xml_dir.exists(): print(f"Error: XML directory not found: {xml_dir}") sys.exit(1) - manifest = build_manifest(xml_dir, version) + manifest = build_manifest(xml_dir, version, src_dir) output_dir.mkdir(parents=True, exist_ok=True) output_file = output_dir / "eql-manifest.json" output_file.write_text(json.dumps(manifest, indent=2) + "\n") @@ -118,6 +216,7 @@ def main(): f" Functions: {counts['functions']} " f"({counts['public']} public, {counts['private']} private)" ) + print(f" Domains: {counts['domains']}") if __name__ == "__main__": From d09852a71ca912b4f9a23db5547052f1dd837005 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 6 Jul 2026 10:43:04 +1000 Subject: [PATCH 3/3] docs: source the domain matrix from the Rust catalog, not SQL parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scalar domains are 100% generated from eql_domains::CATALOG, so read the domain/variant matrix straight from the source of truth via `eql-codegen dump-catalog` (which already emits JSON) instead of parsing the generated *_types.sql. No drift from SQL-format changes, authoritative type tokens (integer/bigint/…, not int4/int8), and the exact SQL operators each domain supports (new `supportedOperators` field). More accurate, too: `_ord` domains now correctly report equality + order — ORE comparison collapses to equality, so `=`/`<>` are supported — which the CHECK-key derivation missed. json.sh emits docs/api/json/eql-catalog.json (cargo run -p eql-codegen dump-catalog) and passes it to the converter. Covers the 48 catalog (scalar) domains; the 3 hand-written jsonb domains (json/jsonb_entry/jsonb_query) are not in the catalog and remain a follow-up. Claude-Session: https://claude.ai/code/session_01CqDNqLSEEkCi7xAJFq7HJA --- tasks/docs/generate/json.sh | 10 +- tasks/docs/generate/test_xml_to_json.py | 61 +++++----- tasks/docs/generate/xml-to-json.py | 141 ++++++++---------------- 3 files changed, 86 insertions(+), 126 deletions(-) diff --git a/tasks/docs/generate/json.sh b/tasks/docs/generate/json.sh index 91316701e..cb0991840 100755 --- a/tasks/docs/generate/json.sh +++ b/tasks/docs/generate/json.sh @@ -13,8 +13,14 @@ if [ ! -d "docs/api/xml" ]; then mise run --output prefix docs:generate fi -# Run converter -mise run --output prefix docs:generate:xml-to-json docs/api/xml docs/api/json "$VERSION" +# Emit the authoritative domain/variant matrix straight from the Rust catalog +# (eql_domains::CATALOG) — the source of truth the generated SQL is rendered from. +mkdir -p docs/api/json +echo "Dumping domain catalog (eql-codegen dump-catalog)..." +cargo run -q -p eql-codegen dump-catalog > docs/api/json/eql-catalog.json + +# Run converter (functions from XML, domains from the catalog dump) +mise run --output prefix docs:generate:xml-to-json docs/api/xml docs/api/json "$VERSION" docs/api/json/eql-catalog.json echo "" echo "✓ JSON manifest: docs/api/json/eql-manifest.json" diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py index 8798d20c8..0070ac1e1 100755 --- a/tasks/docs/generate/test_xml_to_json.py +++ b/tasks/docs/generate/test_xml_to_json.py @@ -18,23 +18,19 @@ _mod = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_mod) build_manifest = _mod.build_manifest -parse_domains = _mod.parse_domains - -DOMAIN_SQL = """DO $$ -BEGIN - --! @brief Encrypted domain eql_v3.text_eq. - IF NOT EXISTS (SELECT 1) THEN - CREATE DOMAIN eql_v3.text_eq AS jsonb - CHECK ( - jsonb_typeof(VALUE) = 'object' - AND VALUE ? 'v' - AND VALUE ? 'i' - AND VALUE ? 'c' - AND VALUE ? 'hm' - AND VALUE->>'v' = '3' - ); - END IF; -END $$;""" +load_domains = _mod.load_domains + +# Shape of `eql-codegen dump-catalog` output. +CATALOG_JSON = """{ + "types": [ + { "token": "text", "is_eq_only": false, "domains": [ + { "segment": "storage", "suffix": "", "supported_ops": [] }, + { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"] }, + { "segment": "search", "suffix": "_search", + "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"] } + ]} + ] +}""" SAMPLE_XML = """ @@ -67,7 +63,10 @@ def test_build_manifest_shape(): with tempfile.TemporaryDirectory() as d: (Path(d) / "hmac.xml").write_text(SAMPLE_XML) - manifest = build_manifest(Path(d), "1.2.3", src_dir=Path(d)) + (Path(d) / "empty-catalog.json").write_text('{"types": []}') + manifest = build_manifest( + Path(d), "1.2.3", catalog_path=Path(d) / "empty-catalog.json" + ) assert manifest["name"] == "eql" assert manifest["version"] == "1.2.3" @@ -94,24 +93,24 @@ def test_skips_index_and_doxyfile(): assert manifest["counts"]["functions"] == 0 -def test_parse_domains(): +def test_load_domains(): with tempfile.TemporaryDirectory() as d: - (Path(d) / "text_types.sql").write_text(DOMAIN_SQL) - domains = parse_domains(Path(d)) + cat = Path(d) / "eql-catalog.json" + cat.write_text(CATALOG_JSON) + domains = load_domains(cat) - assert len(domains) == 1 - dm = domains[0] - assert dm["name"] == "eql_v3.text_eq" - assert dm["type"] == "text" - assert dm["variant"] == "eq" - assert dm["terms"] == ["hm"] # envelope keys (v/i/c) excluded - assert dm["capabilities"] == ["equality"] - assert dm["termFunctions"] == ["eql_v3.hmac_256"] - assert dm["brief"] == "Encrypted domain eql_v3.text_eq." + by_name = {x["name"]: x for x in domains} + assert by_name["public.text"]["capabilities"] == ["storage"] + assert by_name["public.text_eq"]["type"] == "text" + assert by_name["public.text_eq"]["variant"] == "eq" + assert by_name["public.text_eq"]["capabilities"] == ["equality"] + assert by_name["public.text_eq"]["supportedOperators"] == ["=", "<>"] + # text_search carries all three capabilities, derived from its operators. + assert by_name["public.text_search"]["capabilities"] == ["equality", "order", "match"] if __name__ == "__main__": test_build_manifest_shape() test_skips_index_and_doxyfile() - test_parse_domains() + test_load_domains() print("✓ all tests passed") diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py index efc569851..25ec2eeac 100755 --- a/tasks/docs/generate/xml-to-json.py +++ b/tasks/docs/generate/xml-to-json.py @@ -12,16 +12,15 @@ Reuses the extraction in xml-to-markdown.py so the manifest and the Markdown reference can never diverge in how they read the XML. -Doxygen does not extract CREATE DOMAIN, so the manifest also parses the -generated domain SQL to emit the encrypted domain/variant matrix (the core of -the EQL v3 surface) — each domain's capability derived from its CHECK keys. +Doxygen does not extract CREATE DOMAIN, so the manifest reads the encrypted +domain/variant matrix (the core of the EQL v3 surface) straight from the Rust +catalog via `eql-codegen dump-catalog` — authoritative type names + operators. -Usage: xml-to-json.py [output_dir] [version] [sql_src_dir] +Usage: xml-to-json.py [output_dir] [version] [catalog_json] """ import importlib.util import json -import re import sys import xml.etree.ElementTree as ET from pathlib import Path @@ -68,96 +67,50 @@ def _to_entry(func): } -# ── Encrypted domains ──────────────────────────────────────────────────────── -# Doxygen does not extract CREATE DOMAIN, but the domain/variant matrix is the -# core of the EQL v3 surface. The generated `*_types.sql` (source of truth: the -# Rust catalog in crates/eql-domains) encodes each domain's capability -# STRUCTURALLY as the required CHECK keys, so we derive it directly: -# hm = HMAC equality · ob = ORE order · op = OPE order · bf = bloom match · -# sv = STE-vec (JSON). v/i/c are the envelope, not index terms. -_ENVELOPE_KEYS = {"v", "i", "c", "k"} -_TERM_CAPABILITY = { - "hm": "equality", - "ob": "order", - "op": "order", - "bf": "match", - "sv": "json", -} -# Term -> extractor function (from crates/eql-domains/src/term.rs). -_TERM_FUNCTION = { - "hm": "eql_v3.hmac_256", - "ob": "eql_v3.ore_block_256", - "bf": "eql_v3.bloom_filter", -} -# Longest-first so `_ord_ore` wins over `_ord`. -_VARIANT_SUFFIXES = ("_ord_ore", "_ord_ope", "_ord", "_eq", "_match", "_search") - -_DOMAIN_RE = re.compile(r"CREATE DOMAIN eql_v3\.([a-z0-9_]+)\s+AS\s+([a-z_]+)", re.I) -_KEY_RE = re.compile(r"VALUE \? '([a-z0-9]+)'") -_BRIEF_RE = re.compile(r"--!\s*@brief\s+(.*)") - - -def _build_domain(name, base, brief, terms, source_file, line): - scalar_type, variant = name, "" - for suffix in _VARIANT_SUFFIXES: - if name.endswith(suffix): - scalar_type, variant = name[: -len(suffix)], suffix[1:] - break - - capabilities = [] - for term in terms: - cap = _TERM_CAPABILITY.get(term) - if cap and cap not in capabilities: - capabilities.append(cap) - if not capabilities: - capabilities = ["storage"] - - return { - "name": f"eql_v3.{name}", - "type": scalar_type, - "variant": variant, - "base": base, - "brief": brief, - "terms": terms, - "capabilities": capabilities, - "termFunctions": [_TERM_FUNCTION[t] for t in terms if t in _TERM_FUNCTION], - "source": {"file": str(source_file), "line": line}, - } - - -def parse_domains(src_dir: Path) -> list: - """Extract eql_v3 CREATE DOMAIN definitions + their capability from the SQL.""" - if not src_dir.exists(): - print(f"Warning: SQL source dir not found: {src_dir}; skipping domains", file=sys.stderr) +# ── Encrypted domains (from the Rust catalog) ──────────────────────────────── +# The domain/variant matrix is the core of the EQL v3 surface. Rather than parse +# the generated SQL (one step removed, format-fragile), read it straight from the +# source of truth: `eql-codegen dump-catalog` serializes eql_domains::CATALOG — +# authoritative type tokens plus the exact SQL operators each domain supports. +def _capabilities_from_ops(ops): + caps = [] + if any(o in ops for o in ("=", "<>")): + caps.append("equality") + if any(o in ops for o in ("<", "<=", ">", ">=")): + caps.append("order") + if any(o in ops for o in ("@>", "<@")): + caps.append("match") + return caps or ["storage"] + + +def load_domains(catalog_path: Path) -> list: + """Map `eql-codegen dump-catalog` JSON into manifest domain entries.""" + if not catalog_path.exists(): + print(f"Warning: catalog dump not found: {catalog_path}; skipping domains", file=sys.stderr) return [] + catalog = json.loads(catalog_path.read_text()) domains = [] - for sql_file in sorted(src_dir.rglob("*.sql")): - lines = sql_file.read_text().splitlines() - last_brief = "" - for i, line in enumerate(lines): - brief_match = _BRIEF_RE.search(line) - if brief_match: - last_brief = brief_match.group(1).strip() - domain_match = _DOMAIN_RE.search(line) - if not domain_match: - continue - name, base = domain_match.group(1), domain_match.group(2) - # Collect CHECK keys until the block closes or the next domain begins. - keys = [] - for follow in lines[i + 1:]: - if _DOMAIN_RE.search(follow) or re.match(r"\s*\);", follow): - break - keys.extend(_KEY_RE.findall(follow)) - terms = [k for k in dict.fromkeys(keys) if k not in _ENVELOPE_KEYS] - domains.append(_build_domain(name, base, last_brief, terms, sql_file, i + 1)) - last_brief = "" - + for type_entry in catalog.get("types", []): + token = type_entry["token"] + for dom in type_entry["domains"]: + suffix = dom.get("suffix", "") + ops = dom.get("supported_ops", []) + domains.append({ + # v3 user domains live in the `public` schema (public-domain + # migration on eql_v3); the catalog dump emits the bare token. + "name": f"public.{token}{suffix}", + "type": token, + "variant": suffix.lstrip("_"), + "base": "jsonb", + "capabilities": _capabilities_from_ops(ops), + "supportedOperators": ops, + }) domains.sort(key=lambda d: (d["type"], d["name"])) return domains -def build_manifest(xml_dir: Path, version: str, src_dir: Path = Path("src/v3")) -> dict: +def build_manifest(xml_dir: Path, version: str, catalog_path: Path = Path("docs/api/json/eql-catalog.json")) -> dict: functions = [] for xml_file in sorted(xml_dir.glob("*.xml")): if xml_file.name in ("index.xml", "Doxyfile.xml"): @@ -173,13 +126,13 @@ def build_manifest(xml_dir: Path, version: str, src_dir: Path = Path("src/v3")) functions.append(func) functions.sort(key=lambda f: (f["is_private"], f["name"], f["signature"])) - domains = parse_domains(src_dir) + domains = load_domains(catalog_path) return { "$schema": "https://schemas.cipherstash.com/eql/manifest/v1.json", "name": "eql", "version": version, - "generatedFrom": "doxygen-xml + sql-domains", + "generatedFrom": "doxygen-xml + catalog", "counts": { "functions": len(functions), "public": sum(1 for f in functions if not f["is_private"]), @@ -193,19 +146,21 @@ def build_manifest(xml_dir: Path, version: str, src_dir: Path = Path("src/v3")) def main(): if len(sys.argv) < 2: - print("Usage: xml-to-json.py [output_dir] [version] [sql_src_dir]") + print("Usage: xml-to-json.py [output_dir] [version] [catalog_json]") sys.exit(1) xml_dir = Path(sys.argv[1]) output_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("docs/api/json") version = sys.argv[3] if len(sys.argv) > 3 else "DEV" - src_dir = Path(sys.argv[4]) if len(sys.argv) > 4 else Path("src/v3") + catalog_path = ( + Path(sys.argv[4]) if len(sys.argv) > 4 else Path("docs/api/json/eql-catalog.json") + ) if not xml_dir.exists(): print(f"Error: XML directory not found: {xml_dir}") sys.exit(1) - manifest = build_manifest(xml_dir, version, src_dir) + manifest = build_manifest(xml_dir, version, catalog_path) output_dir.mkdir(parents=True, exist_ok=True) output_file = output_dir / "eql-manifest.json" output_file.write_text(json.dumps(manifest, indent=2) + "\n")