fix(docs): recover symbols the manifest extraction was silently dropping

.changeset/manifest-extraction-fixes.md

@@ -0,0 +1,12 @@
+---
+'@cipherstash/eql': patch
+---
+
+**The generated API reference was missing `eql_v3.grouped_value` and `eql_v3.lints()` entirely, and described `public.eql_v3_text_match` as storage-only.** Four extraction bugs, all silent — each one dropped or misdescribed a symbol without failing the docs build.
+
+- **`@example` deleted the symbol it documented.** Doxygen's `\example` takes a *filename*; used as an inline snippet marker it pulled the block out of the member's documentation. The two symbols in the tree that used it were the two missing from the manifest — `eql_v3.grouped_value` lost its whole memberdef, `eql_v3.lints()` was left with an empty description and dropped by the no-documentation guard. Both now use `@code{.sql}` / `@endcode`.
+- **`CREATE AGGREGATE` bodies broke the C++ parse.** The trailing `(sfunc = …, stype = …)` is a second parenthesised group, which C++ has no form for, so Doxygen misread the declaration: with a bare argument type it named the member after the *type* (`grouped_value(jsonb)` was extracted as a function called `jsonb`), and with a schema-qualified one it truncated the argument list mid-body. The input filter now strips aggregate bodies, as it already did for dollar-quoted function bodies, and the extractor reads an aggregate's argument types from its declaration and its return type from `@return`. `min`/`max` signatures are now schema-qualified (`min(public.eql_v3_bigint_ord)`, previously `min(eql_v3_bigint_ord)`) and no longer report a parameter named `public.` or a return type of `CREATE AGGREGATE eql_v3`.
+- **The `match` capability was computed from the pre-3.0.1 operator spelling.** Fuzzy match became `@@` in 3.0.1, but the manifest still tested `@>` / `<@`, so no domain could be assigned `match` at all: `public.eql_v3_text_match` matched no branch and fell through to the storage-only default, and `text_search` silently lost `match`. Containment on encrypted JSON keeps `@>` / `<@` and is now reported as its own `containment` capability rather than conflated with fuzzy match.
+- **`public.eql_v3_json` was in no list at all.** The catalog dump filtered the json family's bare storage domain out of `stevec` as scalar, while `types` iterates `scalar_families()`, which excludes the mixed json family wholesale — so the domain the SQL materializer creates appeared in neither, and never reached the manifest. It is now carried in the json-family inventory with a `scalar` flag, so consumers do not describe it with the SteVec query surface.
+
+Why: the manifest is the machine-readable contract the documentation and downstream agents build against, and every one of these failed open — a missing symbol or a wrong capability reads exactly like a symbol that does not exist or a domain that cannot be searched.

crates/eql-codegen/src/dump.rs

@@ -13,9 +13,18 @@ use serde::Serialize;
 #[derive(Serialize)]
 pub struct CatalogDump {
     pub types: Vec<TypeEntry>,
-    /// The `json` (SteVec) family — `public.eql_v3_json_search` / `public.eql_v3_json_entry` /
-    /// `eql_v3.query_json`. Their SQL is hand-written under `src/v3/json/`; the
-    /// catalog owns only their inventory (scalar-only consumers ignore this field).
+    /// The whole `json` family inventory — the SteVec domains
+    /// (`public.eql_v3_json_search` / `public.eql_v3_json_entry` /
+    /// `eql_v3.query_json`) **and** the bare `public.eql_v3_json` storage
+    /// domain, which is scalar-shaped but belongs to this mixed family. Their
+    /// SQL is hand-written under `src/v3/json/`; the catalog owns only their
+    /// inventory (scalar-only consumers ignore this field).
+    ///
+    /// The storage domain is carried here rather than in `types` because
+    /// `types` is the scalar-*matrix* surface: the catalog-coverage task
+    /// requires a `scalars::<token>::matrix_*` suite for every (type, domain)
+    /// it lists, and the json family has no such matrix. Listing it in neither
+    /// is what dropped `public.eql_v3_json` from the docs manifest entirely.
     pub stevec: Vec<SteVecEntry>,
 }
 
@@ -62,11 +71,11 @@ pub struct TermInfo {
     pub ctor: &'static str,
 }
 
-/// One `json` (SteVec) domain: catalog inventory only — its SQL surface
+/// One `json`-family domain: catalog inventory only — its SQL surface
 /// (CHECK, operators) is hand-written and not derivable from the catalog.
 #[derive(Serialize)]
 pub struct SteVecEntry {
-    /// The bare domain name: `json_search` / `json_entry` / `query_json`.
+    /// The bare domain name: `json` / `json_search` / `json_entry` / `query_json`.
     pub full_name: String,
     /// The installed pg_type typname: the version-prefixed name for the
     /// public-schema column domains (`eql_v3_json_search` /
@@ -79,6 +88,11 @@ pub struct SteVecEntry {
     /// (the sv element type); the `json` container and `query_json` domains
     /// carry no term extractors — see `stevec_terms`.
     pub terms: Vec<TermInfo>,
+    /// True for the family's one scalar-shaped member, the storage-only
+    /// `public.eql_v3_json`. Consumers that describe SteVec capabilities
+    /// (containment, path navigation) must not apply them to this domain: it
+    /// stores and decrypts, nothing more.
+    pub scalar: bool,
 }
 
 fn term_infos(terms: &[Term]) -> Vec<TermInfo> {
@@ -150,20 +164,23 @@ pub fn dump_catalog() -> CatalogDump {
     // The hand-written SteVec (jsonb) family — catalog inventory only. Kept out
     // of `types` so scalar-only consumers (the fixture-coverage task) are
     // unaffected; the docs manifest reads both `types` and `stevec`.
+    // The whole json family, scalar member included. This used to filter out
+    // the bare `public.eql_v3_json` storage domain on the assumption it would
+    // surface via `types` — but `types` iterates `scalar_families()`, which
+    // excludes the mixed json family wholesale, so the domain appeared in
+    // neither list and was missing from the docs manifest even though the SQL
+    // materializer (`families_with_scalar_domains()`) creates it.
     let stevec = eql_domains::JSON
         .domains
         .iter()
-        // The json family is mixed: skip its bare scalar storage domain
-        // (`public.eql_v3_json`) — it is not a SteVec entry (it is generated by
-        // the scalar materializer and dumped, if at all, via `types`).
-        .filter(|d| !d.is_scalar())
         .map(|d| SteVecEntry {
             full_name: d.full_name(eql_domains::JSON.name),
             typname: d.sql_typname(eql_domains::JSON.name),
             name: d.name,
             // Catalog terms are empty for SteVec; hardcode per-domain — only
             // `json_entry` carries extractors (see stevec_terms).
             terms: stevec_terms(d.name),
+            scalar: d.is_scalar(),
         })
         .collect();
 
@@ -238,7 +255,18 @@ mod tests {
     fn stevec_json_family_is_dumped() {
         let dump = dump_catalog();
         let names: Vec<&str> = dump.stevec.iter().map(|e| e.full_name.as_str()).collect();
-        assert_eq!(names, ["json_search", "json_entry", "query_json"]);
+        assert_eq!(names, ["json_search", "json_entry", "query_json", "json"]);
+
+        // The bare storage domain is the family's one scalar member. It is
+        // carried here — not in `types` — and flagged so consumers do not
+        // describe it with the SteVec query surface.
+        let scalars: Vec<&str> = dump
+            .stevec
+            .iter()
+            .filter(|e| e.scalar)
+            .map(|e| e.full_name.as_str())
+            .collect();
+        assert_eq!(scalars, ["json"]);
 
         let by_name = |n: &str| {
             dump.stevec

src/v3/aggregates.sql

@@ -106,15 +106,17 @@ $$;
 --!   encryption of the same plaintext, so any one represents the group) and it
 --!   matches the eql_v2 original.
 --!
---! @example
---! -- Group encrypted rows by encrypted equality and project the encrypted
---! -- column. GROUP BY eql_v3.eq_term(...) groups by the HMAC equality term;
---! -- grouped_value(...) returns a representative ciphertext for each group so
---! -- PostgreSQL does not reject the bare column reference.
+--! Group encrypted rows by encrypted equality and project the encrypted
+--! column. GROUP BY eql_v3.eq_term(...) groups by the HMAC equality term;
+--! grouped_value(...) returns a representative ciphertext for each group so
+--! PostgreSQL does not reject the bare column reference.
+--!
+--! @code{.sql}
 --! SELECT eql_v3.grouped_value(encrypted_foo) AS encrypted_foo,
 --!        count(*)
 --! FROM some_table
 --! GROUP BY eql_v3.eq_term(encrypted_foo);
+--! @endcode
 --!
 --! @see eql_v3_internal.grouped_value_sfunc
 --! @see eql_v3.eq_term

src/v3/lint/lints.sql

@@ -64,13 +64,12 @@
 --!                                type picker, which the schema split exists to
 --!                                prevent. Move it to `eql_v3_internal`.
 --!
---! @example
---! ```
+--! @code{.sql}
 --! SELECT severity, category, object_name, message
 --!   FROM eql_v3.lints()
 --!  WHERE severity = 'error'
 --!  ORDER BY category, object_name;
---! ```
+--! @endcode
 --!
 --! @return SETOF record (severity text, category text, object_name text, message text)
 CREATE OR REPLACE FUNCTION eql_v3.lints()

tasks/docs/doxygen-filter.sh

@@ -10,7 +10,7 @@ if [ "$#" -ne 1 ]; then
   exit 2
 fi
 
-# Prepares SQL for Doxygen's C++ parser. Two transforms:
+# Prepares SQL for Doxygen's C++ parser. Three transforms:
 #
 #  1. `--!` doc comments -> `//!` so Doxygen sees them.
 #  2. Strip dollar-quoted function bodies (`$$ ... $$`), leaving just the
@@ -23,8 +23,49 @@ fi
 #     documentation, so removing them is lossless for the generated reference
 #     and leaves Doxygen only clean `CREATE FUNCTION name(args) RETURNS ...`
 #     declarations to read. Only bare `$$` quoting is used in this codebase.
+#  3. Strip CREATE AGGREGATE definition bodies, for the same reason and with the
+#     same losslessness: `sfunc`/`stype`/`combinefunc`/`parallel` carry no
+#     documentation. The trailing `( ... )` is a SECOND parenthesised group
+#     after the signature, and C++ has no such form, so Doxygen misreads the
+#     whole declaration — differently depending on the argument type:
+#
+#       CREATE AGGREGATE eql_v3.grouped_value(jsonb) (...)
+#         -> a function NAMED `jsonb`. `eql_v3.grouped_value` is absorbed into
+#            the return type and disappears from the docs entirely.
+#       CREATE AGGREGATE eql_v3.min(public.eql_v3_bigint_ord) (...)
+#         -> named `min`, but the argument list is truncated mid-body to
+#            `(public.eql_v3_bigint_ord)(sfunc`.
+#
+#     Reducing each to a single `CREATE AGGREGATE name(argtype);` declaration
+#     recovers the name in both cases and leaves a clean argument list.
 awk '
   /^--!/ { print "//!" substr($0, 4); next }
+  # Emit the signature up to its balanced closing paren, then skip the body.
+  !inagg && /^[[:space:]]*CREATE[[:space:]]+AGGREGATE/ {
+    depth = 0; sig = ""; rest = ""; closed = 0
+    for (i = 1; i <= length($0); i++) {
+      ch = substr($0, i, 1)
+      if (closed) { rest = rest ch; continue }
+      sig = sig ch
+      if (ch == "(") depth++
+      else if (ch == ")" && --depth == 0) closed = 1
+    }
+    # Only when the signature balances on this line (true for every aggregate
+    # in this codebase); otherwise fall through to the generic handling.
+    if (closed) {
+      print sig ";"
+      # Skip the body only if the statement does not also end on this line, so
+      # a single-line `CREATE AGGREGATE x(y) (sfunc = z);` cannot leave the
+      # skip latched and swallow the declarations that follow it.
+      inagg = (index(rest, ";") == 0)
+      next
+    }
+  }
+  # A blank line per skipped body row, never nothing: Doxygen reports positions
+  # in the FILTERED stream, so dropping the rows outright shifts every symbol
+  # after an aggregate (max_sfunc at source line 41 was reported as 36). The
+  # dollar-quoted body stripper below keeps the 1:1 line mapping the same way.
+  inagg { print ""; if (index($0, ");")) inagg = 0; next }
   {
     out = ""
     s = $0

tasks/docs/generate/test_xml_to_json.py

@@ -20,28 +20,37 @@
 build_manifest = _mod.build_manifest
 load_domains = _mod.load_domains
 
-# Shape of `eql-codegen dump-catalog` output.
+# Shape of `eql-codegen dump-catalog` output. Domain names come from `typname`
+# (the installed pg_type name), never from string-building the family name.
 CATALOG_JSON = """{
   "types": [
     { "token": "text", "is_eq_only": false, "domains": [
-      { "segment": "storage", "suffix": "", "supported_ops": [], "terms": [] },
-      { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"],
+      { "segment": "storage", "suffix": "", "typname": "eql_v3_text",
+        "supported_ops": [], "terms": [] },
+      { "segment": "eq", "suffix": "_eq", "typname": "eql_v3_text_eq",
+        "supported_ops": ["=", "<>"],
         "terms": [{"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}] },
-      { "segment": "search", "suffix": "_search",
-        "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"],
+      { "segment": "match", "suffix": "_match", "typname": "eql_v3_text_match",
+        "supported_ops": ["@@"],
+        "terms": [{"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"}] },
+      { "segment": "search", "suffix": "_search", "typname": "eql_v3_text_search",
+        "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@@"],
         "terms": [
           {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"},
           {"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"}
         ] }
     ]}
   ],
   "stevec": [
-    { "full_name": "json", "name": "json", "terms": [] },
-    { "full_name": "jsonb_entry", "name": "entry", "terms": [
-      {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"},
-      {"key": "op", "extractor": "ord_term", "ctor": "ope_cllw"}
-    ] },
-    { "full_name": "query_jsonb", "name": "query", "terms": [] }
+    { "full_name": "json", "typname": "eql_v3_json", "name": "",
+      "terms": [], "scalar": true },
+    { "full_name": "json_search", "typname": "eql_v3_json_search",
+      "name": "json", "terms": [], "scalar": false },
+    { "full_name": "json_entry", "typname": "eql_v3_json_entry", "name": "entry",
+      "terms": [{"key": "op", "extractor": "ord_term", "ctor": "ope_cllw"}],
+      "scalar": false },
+    { "full_name": "query_json", "typname": "query_json", "name": "query",
+      "terms": [], "scalar": false }
   ]
 }"""
 
@@ -120,16 +129,27 @@ def test_load_domains():
     assert by_name["public.eql_v3_text_eq"]["capabilities"] == ["equality"]
     assert by_name["public.eql_v3_text_eq"]["supportedOperators"] == ["=", "<>"]
     assert by_name["public.eql_v3_text_eq"]["termFunctions"] == ["eql_v3.eq_term"]
+    # Fuzzy match is `@@`. A domain whose only operator is `@@` must report the
+    # `match` capability — reading it off the pre-3.0.1 `@>`/`<@` spelling left
+    # text_match describing itself as storage-only.
+    assert by_name["public.eql_v3_text_match"]["supportedOperators"] == ["@@"]
+    assert by_name["public.eql_v3_text_match"]["capabilities"] == ["match"]
     # text_search carries all three capabilities, derived from its operators.
     assert by_name["public.eql_v3_text_search"]["capabilities"] == ["equality", "order", "match"]
-    # SteVec (jsonb) domains come from the `stevec` section. Term extractors are
-    # hardcoded on `jsonb_entry` (the sv element type) ONLY — hm -> eq_term,
-    # op -> ord_term; the `json` container and `query_jsonb` carry none.
-    assert by_name["public.eql_v3_json"]["termFunctions"] == []
-    assert by_name["eql_v3.query_jsonb"]["termFunctions"] == []
-    assert by_name["public.eql_v3_jsonb_entry"]["capabilities"] == ["json"]
-    assert by_name["public.eql_v3_jsonb_entry"]["shape"] == "stevec"
-    assert by_name["public.eql_v3_jsonb_entry"]["termFunctions"] == ["eql_v3.eq_term", "eql_v3.ord_term"]
+    # json-family domains come from the `stevec` section. Term extractors are
+    # hardcoded on `json_entry` (the sv element type) ONLY — op -> ord_term; the
+    # `json_search` document and `query_json` needle carry none.
+    assert by_name["eql_v3.query_json"]["termFunctions"] == []
+    assert by_name["public.eql_v3_json_search"]["capabilities"] == ["json"]
+    assert by_name["public.eql_v3_json_entry"]["capabilities"] == ["json"]
+    assert by_name["public.eql_v3_json_entry"]["shape"] == "stevec"
+    assert by_name["public.eql_v3_json_entry"]["termFunctions"] == ["eql_v3.ord_term"]
+    # The family's bare storage domain is scalar-shaped: it is present, is NOT a
+    # SteVec shape, and does not inherit the `json` query capability.
+    bare = by_name["public.eql_v3_json"]
+    assert bare["capabilities"] == ["storage"]
+    assert bare["termFunctions"] == []
+    assert "shape" not in bare
 
 
 if __name__ == "__main__":