docs: jsonb domains + per-domain terms/extractors in the manifest (closes #365, #366)

crates/eql-codegen/src/dump.rs

@@ -8,10 +8,15 @@
 use eql_domains::Term;
 use serde::Serialize;
 
-/// The catalog surface: every scalar type and its domains.
+/// The catalog surface: every scalar type and its domains, plus the non-scalar
+/// SteVec (`jsonb`) family.
 #[derive(Serialize)]
 pub struct CatalogDump {
     pub types: Vec<TypeEntry>,
+    /// The `jsonb` (SteVec) family — `public.json` / `public.jsonb_entry` /
+    /// `public.jsonb_query`. Their SQL is hand-written under `src/v3/jsonb/`; the
+    /// catalog owns only their inventory (scalar-only consumers ignore this field).
+    pub stevec: Vec<SteVecEntry>,
 }
 
 #[derive(Serialize)]
@@ -36,6 +41,80 @@ pub struct DomainEntry {
     /// SQL operators the domain's terms support, in catalog order. Empty for
     /// the storage domain (no terms).
     pub supported_ops: Vec<&'static str>,
+    /// The index terms this domain carries, with their extractor + SEM ctor.
+    pub terms: Vec<TermInfo>,
+}
+
+/// A domain's index term: payload key + generated extractor + SEM constructor
+/// (from `eql_domains::Term`) — links a domain to its extractor functions.
+#[derive(Serialize)]
+pub struct TermInfo {
+    /// Payload key: `hm` / `ob` / `bf` / `op`.
+    pub key: &'static str,
+    /// Generated extractor function (unqualified): `eq_term` / `ord_term` /
+    /// `match_term` / `ord_ope_term`.
+    pub extractor: &'static str,
+    /// SEM index-term constructor (unqualified): `hmac_256` / `ore_block_256` /
+    /// `bloom_filter` / `ope_cllw`.
+    pub ctor: &'static str,
+}
+
+/// One `jsonb` (SteVec) domain: catalog inventory only — its SQL surface
+/// (CHECK, operators) is hand-written and not derivable from the catalog.
+#[derive(Serialize)]
+pub struct SteVecEntry {
+    /// The bare domain name (resolved under the `public` schema, like a scalar's
+    /// `integer_eq`): `json` / `jsonb_entry` / `jsonb_query`.
+    pub full_name: String,
+    /// The catalog domain name: `json` / `entry` / `query`.
+    pub name: &'static str,
+    /// Index terms for this SteVec domain. Non-empty only for `jsonb_entry`
+    /// (the sv element type); the `json` container and `jsonb_query` domains
+    /// carry no term extractors — see `stevec_terms`.
+    pub terms: Vec<TermInfo>,
+}
+
+fn term_infos(terms: &[Term]) -> Vec<TermInfo> {
+    terms
+        .iter()
+        .map(|t| TermInfo {
+            key: t.json_key(),
+            extractor: t.extractor(),
+            ctor: t.ctor(),
+        })
+        .collect()
+}
+
+/// Index terms for one `jsonb` (SteVec) domain, hardcoded for now.
+///
+/// The catalog does not model per-SteVec-entry terms — `JSONB_DOMAINS` declare
+/// `terms: &[]` and the `shape_and_terms_are_consistent` invariant fails CI if a
+/// non-`Scalar` domain ever gains one — so `term_infos(d.terms)` is provably
+/// empty here. Until the catalog carries them, source the real hand-written
+/// extractors from `src/v3/jsonb/{functions,operators}.sql`.
+///
+/// Terms live on `jsonb_entry` — the sv *element* type — ONLY: `eql_v3.eq_term`
+/// reads `coalesce(hm, oc)` for `=`/`<>`, and `eql_v3.ore_cllw` reads `oc` for
+/// `<`/`<=`/`>`/`>=`. The `json` container and `jsonb_query` domains carry no
+/// term extractors (their surface is containment `@>`/`<@` and path navigation),
+/// so they return no terms. Keyed on the catalog domain name (`json`/`entry`/
+/// `query`).
+fn stevec_terms(name: &str) -> Vec<TermInfo> {
+    if name != "entry" {
+        return Vec::new();
+    }
+    vec![
+        TermInfo {
+            key: "hm",
+            extractor: "eq_term",
+            ctor: "hmac_256",
+        },
+        TermInfo {
+            key: "oc",
+            extractor: "ore_cllw",
+            ctor: "ore_cllw",
+        },
+    ]
 }
 
 /// Build the catalog surface description from `eql_domains::CATALOG`.
@@ -57,6 +136,7 @@ pub fn dump_catalog() -> CatalogDump {
                         format!("_{}", d.name)
                     },
                     supported_ops: Term::operators_for_terms(d.terms),
+                    terms: term_infos(d.terms),
                 })
                 .collect();
             TypeEntry {
@@ -66,7 +146,23 @@ pub fn dump_catalog() -> CatalogDump {
             }
         })
         .collect();
-    CatalogDump { types }
+
+    // The hand-written SteVec (jsonb) family — catalog inventory only. Kept out
+    // of `types` so scalar-only consumers (the fixture-coverage task) are
+    // unaffected; the docs manifest reads both `types` and `stevec`.
+    let stevec = eql_domains::JSONB
+        .domains
+        .iter()
+        .map(|d| SteVecEntry {
+            full_name: d.full_name(eql_domains::JSONB.name),
+            name: d.name,
+            // Catalog terms are empty for SteVec; hardcode per-domain — only
+            // `jsonb_entry` carries extractors (see stevec_terms).
+            terms: stevec_terms(d.name),
+        })
+        .collect();
+
+    CatalogDump { types, stevec }
 }
 
 #[cfg(test)]
@@ -109,6 +205,45 @@ mod tests {
         assert_eq!(ord_ope.supported_ops, ["=", "<>", "<", "<=", ">", ">="]);
     }
 
+    #[test]
+    fn ordered_domain_exposes_its_extractor_and_ctor() {
+        let dump = dump_catalog();
+        let integer = dump.types.iter().find(|t| t.token == "integer").unwrap();
+        let ord = integer.domains.iter().find(|d| d.segment == "ord").unwrap();
+        assert_eq!(ord.terms.len(), 1);
+        assert_eq!(ord.terms[0].key, "ob");
+        assert_eq!(ord.terms[0].extractor, "ord_term");
+        assert_eq!(ord.terms[0].ctor, "ore_block_256");
+    }
+
+    #[test]
+    fn stevec_jsonb_family_is_dumped() {
+        let dump = dump_catalog();
+        let names: Vec<&str> = dump.stevec.iter().map(|e| e.full_name.as_str()).collect();
+        assert_eq!(names, ["json", "jsonb_entry", "jsonb_query"]);
+
+        let by_name = |n: &str| {
+            dump.stevec
+                .iter()
+                .find(|e| e.full_name == n)
+                .unwrap_or_else(|| panic!("{n} present"))
+        };
+
+        // Term extractors live on `jsonb_entry` (the sv element type) ONLY:
+        // `eq_term` (hm/oc equality) + `ore_cllw` (oc ordering).
+        let entry_extractors: Vec<&str> = by_name("jsonb_entry")
+            .terms
+            .iter()
+            .map(|t| t.extractor)
+            .collect();
+        assert_eq!(entry_extractors, ["eq_term", "ore_cllw"]);
+
+        // The `json` container and `jsonb_query` domains carry no term
+        // extractors — their surface is containment (@>, <@) and path nav.
+        assert!(by_name("json").terms.is_empty());
+        assert!(by_name("jsonb_query").terms.is_empty());
+    }
+
     /// Pins the hand-re-derived `suffix` wire field — the one channel with no
     /// other automated reader — so its underscore-prefixed values stay
     /// byte-stable after the catalog dropped the leading underscore from its

tasks/docs/generate/test_xml_to_json.py

@@ -24,11 +24,24 @@
 CATALOG_JSON = """{
   "types": [
     { "token": "text", "is_eq_only": false, "domains": [
-      { "segment": "storage", "suffix": "", "supported_ops": [] },
-      { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"] },
+      { "segment": "storage", "suffix": "", "supported_ops": [], "terms": [] },
+      { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"],
+        "terms": [{"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}] },
       { "segment": "search", "suffix": "_search",
-        "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"] }
+        "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"],
+        "terms": [
+          {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"},
+          {"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"}
+        ] }
     ]}
+  ],
+  "stevec": [
+    { "full_name": "json", "name": "json", "terms": [] },
+    { "full_name": "jsonb_entry", "name": "entry", "terms": [
+      {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"},
+      {"key": "oc", "extractor": "ore_cllw", "ctor": "ore_cllw"}
+    ] },
+    { "full_name": "jsonb_query", "name": "query", "terms": [] }
   ]
 }"""
 
@@ -100,13 +113,23 @@ def test_load_domains():
         domains = load_domains(cat)
 
     by_name = {x["name"]: x for x in domains}
+    # v3 user domains live in `public`; the extractor functions are `eql_v3`.
     assert by_name["public.text"]["capabilities"] == ["storage"]
     assert by_name["public.text_eq"]["type"] == "text"
     assert by_name["public.text_eq"]["variant"] == "eq"
     assert by_name["public.text_eq"]["capabilities"] == ["equality"]
     assert by_name["public.text_eq"]["supportedOperators"] == ["=", "<>"]
+    assert by_name["public.text_eq"]["termFunctions"] == ["eql_v3.eq_term"]
     # text_search carries all three capabilities, derived from its operators.
     assert by_name["public.text_search"]["capabilities"] == ["equality", "order", "match"]
+    # SteVec (jsonb) domains come from the `stevec` section. Term extractors are
+    # hardcoded on `jsonb_entry` (the sv element type) ONLY — hm -> eq_term,
+    # oc -> ore_cllw; the `json` container and `jsonb_query` carry none.
+    assert by_name["public.json"]["termFunctions"] == []
+    assert by_name["public.jsonb_query"]["termFunctions"] == []
+    assert by_name["public.jsonb_entry"]["capabilities"] == ["json"]
+    assert by_name["public.jsonb_entry"]["shape"] == "stevec"
+    assert by_name["public.jsonb_entry"]["termFunctions"] == ["eql_v3.eq_term", "eql_v3.ore_cllw"]
 
 
 if __name__ == "__main__":

tasks/docs/generate/xml-to-json.py

@@ -83,6 +83,11 @@ def _capabilities_from_ops(ops):
     return caps or ["storage"]
 
 
+def _term_functions(terms):
+    """Qualified extractor functions for a domain's terms (e.g. eql_v3.ord_term)."""
+    return [f"eql_v3.{t['extractor']}" for t in terms]
+
+
 def load_domains(catalog_path: Path) -> list:
     """Map `eql-codegen dump-catalog` JSON into manifest domain entries."""
     if not catalog_path.exists():
@@ -91,6 +96,8 @@ def load_domains(catalog_path: Path) -> list:
 
     catalog = json.loads(catalog_path.read_text())
     domains = []
+
+    # Scalar families: capability + operators + extractor functions.
     for type_entry in catalog.get("types", []):
         token = type_entry["token"]
         for dom in type_entry["domains"]:
@@ -105,7 +112,23 @@ def load_domains(catalog_path: Path) -> list:
                 "base": "jsonb",
                 "capabilities": _capabilities_from_ops(ops),
                 "supportedOperators": ops,
+                "termFunctions": _term_functions(dom.get("terms", [])),
             })
+
+    # SteVec (jsonb) family: hand-written SQL, catalog inventory only. Like the
+    # scalar domains these live in `public`; the extractor functions are eql_v3.
+    for entry in catalog.get("stevec", []):
+        domains.append({
+            "name": f"public.{entry['full_name']}",
+            "type": "jsonb",
+            "variant": "",
+            "base": "jsonb",
+            "shape": "stevec",
+            "capabilities": ["json"],
+            "supportedOperators": [],
+            "termFunctions": _term_functions(entry.get("terms", [])),
+        })
+
     domains.sort(key=lambda d: (d["type"], d["name"]))
     return domains