diff --git a/crates/eql-codegen/src/dump.rs b/crates/eql-codegen/src/dump.rs index f8f3cd07b..fb6248d33 100644 --- a/crates/eql-codegen/src/dump.rs +++ b/crates/eql-codegen/src/dump.rs @@ -8,10 +8,15 @@ use eql_domains::Term; use serde::Serialize; -/// The catalog surface: every scalar type and its domains. +/// The catalog surface: every scalar type and its domains, plus the non-scalar +/// SteVec (`jsonb`) family. #[derive(Serialize)] pub struct CatalogDump { pub types: Vec, + /// The `jsonb` (SteVec) family — `public.json` / `public.jsonb_entry` / + /// `public.jsonb_query`. Their SQL is hand-written under `src/v3/jsonb/`; the + /// catalog owns only their inventory (scalar-only consumers ignore this field). + pub stevec: Vec, } #[derive(Serialize)] @@ -36,6 +41,80 @@ pub struct DomainEntry { /// SQL operators the domain's terms support, in catalog order. Empty for /// the storage domain (no terms). pub supported_ops: Vec<&'static str>, + /// The index terms this domain carries, with their extractor + SEM ctor. + pub terms: Vec, +} + +/// A domain's index term: payload key + generated extractor + SEM constructor +/// (from `eql_domains::Term`) — links a domain to its extractor functions. +#[derive(Serialize)] +pub struct TermInfo { + /// Payload key: `hm` / `ob` / `bf` / `op`. + pub key: &'static str, + /// Generated extractor function (unqualified): `eq_term` / `ord_term` / + /// `match_term` / `ord_ope_term`. + pub extractor: &'static str, + /// SEM index-term constructor (unqualified): `hmac_256` / `ore_block_256` / + /// `bloom_filter` / `ope_cllw`. + pub ctor: &'static str, +} + +/// One `jsonb` (SteVec) domain: catalog inventory only — its SQL surface +/// (CHECK, operators) is hand-written and not derivable from the catalog. +#[derive(Serialize)] +pub struct SteVecEntry { + /// The bare domain name (resolved under the `public` schema, like a scalar's + /// `integer_eq`): `json` / `jsonb_entry` / `jsonb_query`. + pub full_name: String, + /// The catalog domain name: `json` / `entry` / `query`. + pub name: &'static str, + /// Index terms for this SteVec domain. Non-empty only for `jsonb_entry` + /// (the sv element type); the `json` container and `jsonb_query` domains + /// carry no term extractors — see `stevec_terms`. + pub terms: Vec, +} + +fn term_infos(terms: &[Term]) -> Vec { + terms + .iter() + .map(|t| TermInfo { + key: t.json_key(), + extractor: t.extractor(), + ctor: t.ctor(), + }) + .collect() +} + +/// Index terms for one `jsonb` (SteVec) domain, hardcoded for now. +/// +/// The catalog does not model per-SteVec-entry terms — `JSONB_DOMAINS` declare +/// `terms: &[]` and the `shape_and_terms_are_consistent` invariant fails CI if a +/// non-`Scalar` domain ever gains one — so `term_infos(d.terms)` is provably +/// empty here. Until the catalog carries them, source the real hand-written +/// extractors from `src/v3/jsonb/{functions,operators}.sql`. +/// +/// Terms live on `jsonb_entry` — the sv *element* type — ONLY: `eql_v3.eq_term` +/// reads `coalesce(hm, oc)` for `=`/`<>`, and `eql_v3.ore_cllw` reads `oc` for +/// `<`/`<=`/`>`/`>=`. The `json` container and `jsonb_query` domains carry no +/// term extractors (their surface is containment `@>`/`<@` and path navigation), +/// so they return no terms. Keyed on the catalog domain name (`json`/`entry`/ +/// `query`). +fn stevec_terms(name: &str) -> Vec { + if name != "entry" { + return Vec::new(); + } + vec![ + TermInfo { + key: "hm", + extractor: "eq_term", + ctor: "hmac_256", + }, + TermInfo { + key: "oc", + extractor: "ore_cllw", + ctor: "ore_cllw", + }, + ] } /// Build the catalog surface description from `eql_domains::CATALOG`. @@ -57,6 +136,7 @@ pub fn dump_catalog() -> CatalogDump { format!("_{}", d.name) }, supported_ops: Term::operators_for_terms(d.terms), + terms: term_infos(d.terms), }) .collect(); TypeEntry { @@ -66,7 +146,23 @@ pub fn dump_catalog() -> CatalogDump { } }) .collect(); - CatalogDump { types } + + // The hand-written SteVec (jsonb) family — catalog inventory only. Kept out + // of `types` so scalar-only consumers (the fixture-coverage task) are + // unaffected; the docs manifest reads both `types` and `stevec`. + let stevec = eql_domains::JSONB + .domains + .iter() + .map(|d| SteVecEntry { + full_name: d.full_name(eql_domains::JSONB.name), + name: d.name, + // Catalog terms are empty for SteVec; hardcode per-domain — only + // `jsonb_entry` carries extractors (see stevec_terms). + terms: stevec_terms(d.name), + }) + .collect(); + + CatalogDump { types, stevec } } #[cfg(test)] @@ -109,6 +205,45 @@ mod tests { assert_eq!(ord_ope.supported_ops, ["=", "<>", "<", "<=", ">", ">="]); } + #[test] + fn ordered_domain_exposes_its_extractor_and_ctor() { + let dump = dump_catalog(); + let integer = dump.types.iter().find(|t| t.token == "integer").unwrap(); + let ord = integer.domains.iter().find(|d| d.segment == "ord").unwrap(); + assert_eq!(ord.terms.len(), 1); + assert_eq!(ord.terms[0].key, "ob"); + assert_eq!(ord.terms[0].extractor, "ord_term"); + assert_eq!(ord.terms[0].ctor, "ore_block_256"); + } + + #[test] + fn stevec_jsonb_family_is_dumped() { + let dump = dump_catalog(); + let names: Vec<&str> = dump.stevec.iter().map(|e| e.full_name.as_str()).collect(); + assert_eq!(names, ["json", "jsonb_entry", "jsonb_query"]); + + let by_name = |n: &str| { + dump.stevec + .iter() + .find(|e| e.full_name == n) + .unwrap_or_else(|| panic!("{n} present")) + }; + + // Term extractors live on `jsonb_entry` (the sv element type) ONLY: + // `eq_term` (hm/oc equality) + `ore_cllw` (oc ordering). + let entry_extractors: Vec<&str> = by_name("jsonb_entry") + .terms + .iter() + .map(|t| t.extractor) + .collect(); + assert_eq!(entry_extractors, ["eq_term", "ore_cllw"]); + + // The `json` container and `jsonb_query` domains carry no term + // extractors — their surface is containment (@>, <@) and path nav. + assert!(by_name("json").terms.is_empty()); + assert!(by_name("jsonb_query").terms.is_empty()); + } + /// Pins the hand-re-derived `suffix` wire field — the one channel with no /// other automated reader — so its underscore-prefixed values stay /// byte-stable after the catalog dropped the leading underscore from its diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py index 0070ac1e1..52930da2a 100755 --- a/tasks/docs/generate/test_xml_to_json.py +++ b/tasks/docs/generate/test_xml_to_json.py @@ -24,11 +24,24 @@ CATALOG_JSON = """{ "types": [ { "token": "text", "is_eq_only": false, "domains": [ - { "segment": "storage", "suffix": "", "supported_ops": [] }, - { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"] }, + { "segment": "storage", "suffix": "", "supported_ops": [], "terms": [] }, + { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"], + "terms": [{"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}] }, { "segment": "search", "suffix": "_search", - "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"] } + "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"], + "terms": [ + {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}, + {"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"} + ] } ]} + ], + "stevec": [ + { "full_name": "json", "name": "json", "terms": [] }, + { "full_name": "jsonb_entry", "name": "entry", "terms": [ + {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}, + {"key": "oc", "extractor": "ore_cllw", "ctor": "ore_cllw"} + ] }, + { "full_name": "jsonb_query", "name": "query", "terms": [] } ] }""" @@ -100,13 +113,23 @@ def test_load_domains(): domains = load_domains(cat) by_name = {x["name"]: x for x in domains} + # v3 user domains live in `public`; the extractor functions are `eql_v3`. assert by_name["public.text"]["capabilities"] == ["storage"] assert by_name["public.text_eq"]["type"] == "text" assert by_name["public.text_eq"]["variant"] == "eq" assert by_name["public.text_eq"]["capabilities"] == ["equality"] assert by_name["public.text_eq"]["supportedOperators"] == ["=", "<>"] + assert by_name["public.text_eq"]["termFunctions"] == ["eql_v3.eq_term"] # text_search carries all three capabilities, derived from its operators. assert by_name["public.text_search"]["capabilities"] == ["equality", "order", "match"] + # SteVec (jsonb) domains come from the `stevec` section. Term extractors are + # hardcoded on `jsonb_entry` (the sv element type) ONLY — hm -> eq_term, + # oc -> ore_cllw; the `json` container and `jsonb_query` carry none. + assert by_name["public.json"]["termFunctions"] == [] + assert by_name["public.jsonb_query"]["termFunctions"] == [] + assert by_name["public.jsonb_entry"]["capabilities"] == ["json"] + assert by_name["public.jsonb_entry"]["shape"] == "stevec" + assert by_name["public.jsonb_entry"]["termFunctions"] == ["eql_v3.eq_term", "eql_v3.ore_cllw"] if __name__ == "__main__": diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py index 25ec2eeac..2d2f2ff3d 100755 --- a/tasks/docs/generate/xml-to-json.py +++ b/tasks/docs/generate/xml-to-json.py @@ -83,6 +83,11 @@ def _capabilities_from_ops(ops): return caps or ["storage"] +def _term_functions(terms): + """Qualified extractor functions for a domain's terms (e.g. eql_v3.ord_term).""" + return [f"eql_v3.{t['extractor']}" for t in terms] + + def load_domains(catalog_path: Path) -> list: """Map `eql-codegen dump-catalog` JSON into manifest domain entries.""" if not catalog_path.exists(): @@ -91,6 +96,8 @@ def load_domains(catalog_path: Path) -> list: catalog = json.loads(catalog_path.read_text()) domains = [] + + # Scalar families: capability + operators + extractor functions. for type_entry in catalog.get("types", []): token = type_entry["token"] for dom in type_entry["domains"]: @@ -105,7 +112,23 @@ def load_domains(catalog_path: Path) -> list: "base": "jsonb", "capabilities": _capabilities_from_ops(ops), "supportedOperators": ops, + "termFunctions": _term_functions(dom.get("terms", [])), }) + + # SteVec (jsonb) family: hand-written SQL, catalog inventory only. Like the + # scalar domains these live in `public`; the extractor functions are eql_v3. + for entry in catalog.get("stevec", []): + domains.append({ + "name": f"public.{entry['full_name']}", + "type": "jsonb", + "variant": "", + "base": "jsonb", + "shape": "stevec", + "capabilities": ["json"], + "supportedOperators": [], + "termFunctions": _term_functions(entry.get("terms", [])), + }) + domains.sort(key=lambda d: (d["type"], d["name"])) return domains