diff --git a/.changeset/manifest-extraction-fixes.md b/.changeset/manifest-extraction-fixes.md new file mode 100644 index 000000000..2b61356bd --- /dev/null +++ b/.changeset/manifest-extraction-fixes.md @@ -0,0 +1,12 @@ +--- +'@cipherstash/eql': patch +--- + +**The generated API reference was missing `eql_v3.grouped_value` and `eql_v3.lints()` entirely, and described `public.eql_v3_text_match` as storage-only.** Four extraction bugs, all silent — each one dropped or misdescribed a symbol without failing the docs build. + +- **`@example` deleted the symbol it documented.** Doxygen's `\example` takes a *filename*; used as an inline snippet marker it pulled the block out of the member's documentation. The two symbols in the tree that used it were the two missing from the manifest — `eql_v3.grouped_value` lost its whole memberdef, `eql_v3.lints()` was left with an empty description and dropped by the no-documentation guard. Both now use `@code{.sql}` / `@endcode`. +- **`CREATE AGGREGATE` bodies broke the C++ parse.** The trailing `(sfunc = …, stype = …)` is a second parenthesised group, which C++ has no form for, so Doxygen misread the declaration: with a bare argument type it named the member after the *type* (`grouped_value(jsonb)` was extracted as a function called `jsonb`), and with a schema-qualified one it truncated the argument list mid-body. The input filter now strips aggregate bodies, as it already did for dollar-quoted function bodies, and the extractor reads an aggregate's argument types from its declaration and its return type from `@return`. `min`/`max` signatures are now schema-qualified (`min(public.eql_v3_bigint_ord)`, previously `min(eql_v3_bigint_ord)`) and no longer report a parameter named `public.` or a return type of `CREATE AGGREGATE eql_v3`. +- **The `match` capability was computed from the pre-3.0.1 operator spelling.** Fuzzy match became `@@` in 3.0.1, but the manifest still tested `@>` / `<@`, so no domain could be assigned `match` at all: `public.eql_v3_text_match` matched no branch and fell through to the storage-only default, and `text_search` silently lost `match`. Containment on encrypted JSON keeps `@>` / `<@` and is now reported as its own `containment` capability rather than conflated with fuzzy match. +- **`public.eql_v3_json` was in no list at all.** The catalog dump filtered the json family's bare storage domain out of `stevec` as scalar, while `types` iterates `scalar_families()`, which excludes the mixed json family wholesale — so the domain the SQL materializer creates appeared in neither, and never reached the manifest. It is now carried in the json-family inventory with a `scalar` flag, so consumers do not describe it with the SteVec query surface. + +Why: the manifest is the machine-readable contract the documentation and downstream agents build against, and every one of these failed open — a missing symbol or a wrong capability reads exactly like a symbol that does not exist or a domain that cannot be searched. diff --git a/crates/eql-codegen/src/dump.rs b/crates/eql-codegen/src/dump.rs index 65747109e..63868342a 100644 --- a/crates/eql-codegen/src/dump.rs +++ b/crates/eql-codegen/src/dump.rs @@ -13,9 +13,18 @@ use serde::Serialize; #[derive(Serialize)] pub struct CatalogDump { pub types: Vec, - /// The `json` (SteVec) family — `public.eql_v3_json_search` / `public.eql_v3_json_entry` / - /// `eql_v3.query_json`. Their SQL is hand-written under `src/v3/json/`; the - /// catalog owns only their inventory (scalar-only consumers ignore this field). + /// The whole `json` family inventory — the SteVec domains + /// (`public.eql_v3_json_search` / `public.eql_v3_json_entry` / + /// `eql_v3.query_json`) **and** the bare `public.eql_v3_json` storage + /// domain, which is scalar-shaped but belongs to this mixed family. Their + /// SQL is hand-written under `src/v3/json/`; the catalog owns only their + /// inventory (scalar-only consumers ignore this field). + /// + /// The storage domain is carried here rather than in `types` because + /// `types` is the scalar-*matrix* surface: the catalog-coverage task + /// requires a `scalars::::matrix_*` suite for every (type, domain) + /// it lists, and the json family has no such matrix. Listing it in neither + /// is what dropped `public.eql_v3_json` from the docs manifest entirely. pub stevec: Vec, } @@ -62,11 +71,11 @@ pub struct TermInfo { pub ctor: &'static str, } -/// One `json` (SteVec) domain: catalog inventory only — its SQL surface +/// One `json`-family domain: catalog inventory only — its SQL surface /// (CHECK, operators) is hand-written and not derivable from the catalog. #[derive(Serialize)] pub struct SteVecEntry { - /// The bare domain name: `json_search` / `json_entry` / `query_json`. + /// The bare domain name: `json` / `json_search` / `json_entry` / `query_json`. pub full_name: String, /// The installed pg_type typname: the version-prefixed name for the /// public-schema column domains (`eql_v3_json_search` / @@ -79,6 +88,11 @@ pub struct SteVecEntry { /// (the sv element type); the `json` container and `query_json` domains /// carry no term extractors — see `stevec_terms`. pub terms: Vec, + /// True for the family's one scalar-shaped member, the storage-only + /// `public.eql_v3_json`. Consumers that describe SteVec capabilities + /// (containment, path navigation) must not apply them to this domain: it + /// stores and decrypts, nothing more. + pub scalar: bool, } fn term_infos(terms: &[Term]) -> Vec { @@ -150,13 +164,15 @@ pub fn dump_catalog() -> CatalogDump { // The hand-written SteVec (jsonb) family — catalog inventory only. Kept out // of `types` so scalar-only consumers (the fixture-coverage task) are // unaffected; the docs manifest reads both `types` and `stevec`. + // The whole json family, scalar member included. This used to filter out + // the bare `public.eql_v3_json` storage domain on the assumption it would + // surface via `types` — but `types` iterates `scalar_families()`, which + // excludes the mixed json family wholesale, so the domain appeared in + // neither list and was missing from the docs manifest even though the SQL + // materializer (`families_with_scalar_domains()`) creates it. let stevec = eql_domains::JSON .domains .iter() - // The json family is mixed: skip its bare scalar storage domain - // (`public.eql_v3_json`) — it is not a SteVec entry (it is generated by - // the scalar materializer and dumped, if at all, via `types`). - .filter(|d| !d.is_scalar()) .map(|d| SteVecEntry { full_name: d.full_name(eql_domains::JSON.name), typname: d.sql_typname(eql_domains::JSON.name), @@ -164,6 +180,7 @@ pub fn dump_catalog() -> CatalogDump { // Catalog terms are empty for SteVec; hardcode per-domain — only // `json_entry` carries extractors (see stevec_terms). terms: stevec_terms(d.name), + scalar: d.is_scalar(), }) .collect(); @@ -238,7 +255,18 @@ mod tests { fn stevec_json_family_is_dumped() { let dump = dump_catalog(); let names: Vec<&str> = dump.stevec.iter().map(|e| e.full_name.as_str()).collect(); - assert_eq!(names, ["json_search", "json_entry", "query_json"]); + assert_eq!(names, ["json_search", "json_entry", "query_json", "json"]); + + // The bare storage domain is the family's one scalar member. It is + // carried here — not in `types` — and flagged so consumers do not + // describe it with the SteVec query surface. + let scalars: Vec<&str> = dump + .stevec + .iter() + .filter(|e| e.scalar) + .map(|e| e.full_name.as_str()) + .collect(); + assert_eq!(scalars, ["json"]); let by_name = |n: &str| { dump.stevec diff --git a/src/v3/aggregates.sql b/src/v3/aggregates.sql index 8c44916c8..93402386f 100644 --- a/src/v3/aggregates.sql +++ b/src/v3/aggregates.sql @@ -106,15 +106,17 @@ $$; --! encryption of the same plaintext, so any one represents the group) and it --! matches the eql_v2 original. --! ---! @example ---! -- Group encrypted rows by encrypted equality and project the encrypted ---! -- column. GROUP BY eql_v3.eq_term(...) groups by the HMAC equality term; ---! -- grouped_value(...) returns a representative ciphertext for each group so ---! -- PostgreSQL does not reject the bare column reference. +--! Group encrypted rows by encrypted equality and project the encrypted +--! column. GROUP BY eql_v3.eq_term(...) groups by the HMAC equality term; +--! grouped_value(...) returns a representative ciphertext for each group so +--! PostgreSQL does not reject the bare column reference. +--! +--! @code{.sql} --! SELECT eql_v3.grouped_value(encrypted_foo) AS encrypted_foo, --! count(*) --! FROM some_table --! GROUP BY eql_v3.eq_term(encrypted_foo); +--! @endcode --! --! @see eql_v3_internal.grouped_value_sfunc --! @see eql_v3.eq_term diff --git a/src/v3/lint/lints.sql b/src/v3/lint/lints.sql index 650c89d3c..69878bf80 100644 --- a/src/v3/lint/lints.sql +++ b/src/v3/lint/lints.sql @@ -64,13 +64,12 @@ --! type picker, which the schema split exists to --! prevent. Move it to `eql_v3_internal`. --! ---! @example ---! ``` +--! @code{.sql} --! SELECT severity, category, object_name, message --! FROM eql_v3.lints() --! WHERE severity = 'error' --! ORDER BY category, object_name; ---! ``` +--! @endcode --! --! @return SETOF record (severity text, category text, object_name text, message text) CREATE OR REPLACE FUNCTION eql_v3.lints() diff --git a/tasks/docs/doxygen-filter.sh b/tasks/docs/doxygen-filter.sh index b64bd806a..0b5313bdb 100755 --- a/tasks/docs/doxygen-filter.sh +++ b/tasks/docs/doxygen-filter.sh @@ -10,7 +10,7 @@ if [ "$#" -ne 1 ]; then exit 2 fi -# Prepares SQL for Doxygen's C++ parser. Two transforms: +# Prepares SQL for Doxygen's C++ parser. Three transforms: # # 1. `--!` doc comments -> `//!` so Doxygen sees them. # 2. Strip dollar-quoted function bodies (`$$ ... $$`), leaving just the @@ -23,8 +23,49 @@ fi # documentation, so removing them is lossless for the generated reference # and leaves Doxygen only clean `CREATE FUNCTION name(args) RETURNS ...` # declarations to read. Only bare `$$` quoting is used in this codebase. +# 3. Strip CREATE AGGREGATE definition bodies, for the same reason and with the +# same losslessness: `sfunc`/`stype`/`combinefunc`/`parallel` carry no +# documentation. The trailing `( ... )` is a SECOND parenthesised group +# after the signature, and C++ has no such form, so Doxygen misreads the +# whole declaration — differently depending on the argument type: +# +# CREATE AGGREGATE eql_v3.grouped_value(jsonb) (...) +# -> a function NAMED `jsonb`. `eql_v3.grouped_value` is absorbed into +# the return type and disappears from the docs entirely. +# CREATE AGGREGATE eql_v3.min(public.eql_v3_bigint_ord) (...) +# -> named `min`, but the argument list is truncated mid-body to +# `(public.eql_v3_bigint_ord)(sfunc`. +# +# Reducing each to a single `CREATE AGGREGATE name(argtype);` declaration +# recovers the name in both cases and leaves a clean argument list. awk ' /^--!/ { print "//!" substr($0, 4); next } + # Emit the signature up to its balanced closing paren, then skip the body. + !inagg && /^[[:space:]]*CREATE[[:space:]]+AGGREGATE/ { + depth = 0; sig = ""; rest = ""; closed = 0 + for (i = 1; i <= length($0); i++) { + ch = substr($0, i, 1) + if (closed) { rest = rest ch; continue } + sig = sig ch + if (ch == "(") depth++ + else if (ch == ")" && --depth == 0) closed = 1 + } + # Only when the signature balances on this line (true for every aggregate + # in this codebase); otherwise fall through to the generic handling. + if (closed) { + print sig ";" + # Skip the body only if the statement does not also end on this line, so + # a single-line `CREATE AGGREGATE x(y) (sfunc = z);` cannot leave the + # skip latched and swallow the declarations that follow it. + inagg = (index(rest, ";") == 0) + next + } + } + # A blank line per skipped body row, never nothing: Doxygen reports positions + # in the FILTERED stream, so dropping the rows outright shifts every symbol + # after an aggregate (max_sfunc at source line 41 was reported as 36). The + # dollar-quoted body stripper below keeps the 1:1 line mapping the same way. + inagg { print ""; if (index($0, ");")) inagg = 0; next } { out = "" s = $0 diff --git a/tasks/docs/generate/test_xml_to_json.py b/tasks/docs/generate/test_xml_to_json.py index 06349de8a..4fbb5b2e5 100755 --- a/tasks/docs/generate/test_xml_to_json.py +++ b/tasks/docs/generate/test_xml_to_json.py @@ -20,15 +20,21 @@ build_manifest = _mod.build_manifest load_domains = _mod.load_domains -# Shape of `eql-codegen dump-catalog` output. +# Shape of `eql-codegen dump-catalog` output. Domain names come from `typname` +# (the installed pg_type name), never from string-building the family name. CATALOG_JSON = """{ "types": [ { "token": "text", "is_eq_only": false, "domains": [ - { "segment": "storage", "suffix": "", "supported_ops": [], "terms": [] }, - { "segment": "eq", "suffix": "_eq", "supported_ops": ["=", "<>"], + { "segment": "storage", "suffix": "", "typname": "eql_v3_text", + "supported_ops": [], "terms": [] }, + { "segment": "eq", "suffix": "_eq", "typname": "eql_v3_text_eq", + "supported_ops": ["=", "<>"], "terms": [{"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}] }, - { "segment": "search", "suffix": "_search", - "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@>", "<@"], + { "segment": "match", "suffix": "_match", "typname": "eql_v3_text_match", + "supported_ops": ["@@"], + "terms": [{"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"}] }, + { "segment": "search", "suffix": "_search", "typname": "eql_v3_text_search", + "supported_ops": ["=", "<>", "<", "<=", ">", ">=", "@@"], "terms": [ {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}, {"key": "bf", "extractor": "match_term", "ctor": "bloom_filter"} @@ -36,12 +42,15 @@ ]} ], "stevec": [ - { "full_name": "json", "name": "json", "terms": [] }, - { "full_name": "jsonb_entry", "name": "entry", "terms": [ - {"key": "hm", "extractor": "eq_term", "ctor": "hmac_256"}, - {"key": "op", "extractor": "ord_term", "ctor": "ope_cllw"} - ] }, - { "full_name": "query_jsonb", "name": "query", "terms": [] } + { "full_name": "json", "typname": "eql_v3_json", "name": "", + "terms": [], "scalar": true }, + { "full_name": "json_search", "typname": "eql_v3_json_search", + "name": "json", "terms": [], "scalar": false }, + { "full_name": "json_entry", "typname": "eql_v3_json_entry", "name": "entry", + "terms": [{"key": "op", "extractor": "ord_term", "ctor": "ope_cllw"}], + "scalar": false }, + { "full_name": "query_json", "typname": "query_json", "name": "query", + "terms": [], "scalar": false } ] }""" @@ -120,16 +129,27 @@ def test_load_domains(): assert by_name["public.eql_v3_text_eq"]["capabilities"] == ["equality"] assert by_name["public.eql_v3_text_eq"]["supportedOperators"] == ["=", "<>"] assert by_name["public.eql_v3_text_eq"]["termFunctions"] == ["eql_v3.eq_term"] + # Fuzzy match is `@@`. A domain whose only operator is `@@` must report the + # `match` capability — reading it off the pre-3.0.1 `@>`/`<@` spelling left + # text_match describing itself as storage-only. + assert by_name["public.eql_v3_text_match"]["supportedOperators"] == ["@@"] + assert by_name["public.eql_v3_text_match"]["capabilities"] == ["match"] # text_search carries all three capabilities, derived from its operators. assert by_name["public.eql_v3_text_search"]["capabilities"] == ["equality", "order", "match"] - # SteVec (jsonb) domains come from the `stevec` section. Term extractors are - # hardcoded on `jsonb_entry` (the sv element type) ONLY — hm -> eq_term, - # op -> ord_term; the `json` container and `query_jsonb` carry none. - assert by_name["public.eql_v3_json"]["termFunctions"] == [] - assert by_name["eql_v3.query_jsonb"]["termFunctions"] == [] - assert by_name["public.eql_v3_jsonb_entry"]["capabilities"] == ["json"] - assert by_name["public.eql_v3_jsonb_entry"]["shape"] == "stevec" - assert by_name["public.eql_v3_jsonb_entry"]["termFunctions"] == ["eql_v3.eq_term", "eql_v3.ord_term"] + # json-family domains come from the `stevec` section. Term extractors are + # hardcoded on `json_entry` (the sv element type) ONLY — op -> ord_term; the + # `json_search` document and `query_json` needle carry none. + assert by_name["eql_v3.query_json"]["termFunctions"] == [] + assert by_name["public.eql_v3_json_search"]["capabilities"] == ["json"] + assert by_name["public.eql_v3_json_entry"]["capabilities"] == ["json"] + assert by_name["public.eql_v3_json_entry"]["shape"] == "stevec" + assert by_name["public.eql_v3_json_entry"]["termFunctions"] == ["eql_v3.ord_term"] + # The family's bare storage domain is scalar-shaped: it is present, is NOT a + # SteVec shape, and does not inherit the `json` query capability. + bare = by_name["public.eql_v3_json"] + assert bare["capabilities"] == ["storage"] + assert bare["termFunctions"] == [] + assert "shape" not in bare if __name__ == "__main__": diff --git a/tasks/docs/generate/test_xml_to_markdown.py b/tasks/docs/generate/test_xml_to_markdown.py index 3efef0a73..c3b907d84 100755 --- a/tasks/docs/generate/test_xml_to_markdown.py +++ b/tasks/docs/generate/test_xml_to_markdown.py @@ -199,6 +199,99 @@ def memberdef(schema, fn): print("✓ Internal-schema private detection test passed") +def test_returns_table_type_is_not_truncated(): + """A set-returning `RETURNS TABLE (...)` yields `TABLE`, not a fragment. + + Doxygen reads the column list as a C++ argument list and truncates it at the + first comma, so argsstring arrives as `() RETURNS TABLE(severity text`. + Matching to the next space published "TABLE(severity" as the return type of + eql_v3.lints(). The column list cannot be recovered from the XML — the + truncation is at the comma, not a line break — but @return spells it out, so + the type must at least stop cleanly at the paren. + """ + from xml.etree import ElementTree as ET + + process_function = _load_process_function() + + lints = process_function(ET.fromstring(''' + + lints + CREATE OR REPLACE FUNCTION eql_v3 + () RETURNS TABLE(severity text + EQL lint results. + + SETOF record (severity text, category text) + + + ''')) + assert lints is not None, "lints should be extracted" + assert lints["return_type"] == "`TABLE`", ( + f'expected a clean `TABLE`, got {lints["return_type"]!r}' + ) + + # Ordinary scalar return types are untouched by the paren stop. + scalar = process_function(ET.fromstring(''' + + eq_term + CREATE FUNCTION eql_v3 + (val jsonb) RETURNS bytea + Extract a term. + + + ''')) + assert scalar["return_type"] == "`bytea`", scalar["return_type"] + + print("✓ RETURNS TABLE return-type test passed") + +def test_filter_preserves_line_numbering(): + """The Doxygen input filter emits one output line per input line. + + Doxygen reports source positions against the FILTERED stream, so a + transform that drops rows silently shifts every symbol after it. Skipping + CREATE AGGREGATE bodies without padding moved max_sfunc from line 41 to 36. + """ + import subprocess + import tempfile + + sql = '''--! @brief State function. +CREATE FUNCTION eql_v3_internal.min_sfunc(state jsonb, value jsonb) +RETURNS jsonb +LANGUAGE sql AS $$ + SELECT 1 +$$; + +--! @brief min aggregate. +CREATE AGGREGATE eql_v3.min(public.eql_v3_bigint_ord) ( + sfunc = eql_v3_internal.min_sfunc, + stype = public.eql_v3_bigint_ord, + parallel = safe +); + +--! @brief Trailing function whose line number must not shift. +CREATE FUNCTION eql_v3.after(a jsonb) RETURNS jsonb +LANGUAGE sql AS $$ SELECT a $$; +''' + filter_script = Path(__file__).resolve().parents[1] / "doxygen-filter.sh" + with tempfile.TemporaryDirectory() as d: + path = Path(d) / "aggregates.sql" + path.write_text(sql) + out = subprocess.run( + [str(filter_script), str(path)], capture_output=True, text=True, check=True + ).stdout + + assert out.count("\n") == sql.count("\n"), ( + f"filter changed line count: {sql.count(chr(10))} in, {out.count(chr(10))} out" + ) + # The declaration after the aggregate must still sit on its original line. + marker = "CREATE FUNCTION eql_v3.after" + src_line = next(i for i, l in enumerate(sql.splitlines()) if l.startswith(marker)) + out_lines = out.splitlines() + assert out_lines[src_line].startswith(marker), ( + f"line {src_line + 1} shifted, got: {out_lines[src_line]!r}" + ) + + print("✓ Filter line-numbering test passed") + def test_schema_name_misparse_is_skipped(): """Name-dropped CREATE FUNCTION mis-parses are skipped, operators are kept. @@ -251,6 +344,8 @@ def test_schema_name_misparse_is_skipped(): test_param_name_type_swap() test_schema_qualified_type() test_internal_schema_is_private() + test_returns_table_type_is_not_truncated() + test_filter_preserves_line_numbering() test_schema_name_misparse_is_skipped() print("\n✅ All tests passed!") diff --git a/tasks/docs/generate/xml-to-json.py b/tasks/docs/generate/xml-to-json.py index 34ddcde35..8e8b1af80 100755 --- a/tasks/docs/generate/xml-to-json.py +++ b/tasks/docs/generate/xml-to-json.py @@ -78,8 +78,18 @@ def _capabilities_from_ops(ops): caps.append("equality") if any(o in ops for o in ("<", "<=", ">", ">=")): caps.append("order") - if any(o in ops for o in ("@>", "<@")): + # Text fuzzy match is `@@`. It was spelled `@>` / `<@` until EQL 3.0.1 + # renamed it; testing the old operators here meant NO domain could be + # assigned `match` any more. `text_match` (whose only operator is `@@`) + # matched no branch at all and fell through to the `storage` default, + # describing the fuzzy-match domain as storage-only, and `text_search` + # silently lost `match` from its capability list. + if "@@" in ops: caps.append("match") + # Containment on encrypted JSON keeps `@>` / `<@`; it is a distinct + # capability from text fuzzy match, not a spelling of it. + if any(o in ops for o in ("@>", "<@")): + caps.append("containment") return caps or ["storage"] @@ -117,22 +127,30 @@ def load_domains(catalog_path: Path) -> list: "termFunctions": _term_functions(dom.get("terms", [])), }) - # SteVec (jsonb) family: hand-written SQL, catalog inventory only. The - # column-type domains (`json`, `jsonb_entry`) live in `public`; the - # containment needle (`query_jsonb`) is a query operand, never a column - # type, and lives in `eql_v3`. Extractor functions are eql_v3. + # The json family: hand-written SQL, catalog inventory only. The column-type + # domains (`eql_v3_json`, `eql_v3_json_search`, `eql_v3_json_entry`) live in + # `public`; the containment needle (`query_json`) is a query operand, never a + # column type, and lives in `eql_v3`. Extractor functions are eql_v3. + # + # The family is mixed. Its bare `public.eql_v3_json` is a storage-only scalar + # domain — it stores and decrypts, and carries no query surface — so it must + # not inherit the SteVec `json` capability that the searchable document, + # entry, and needle domains carry. for entry in catalog.get("stevec", []): schema = "eql_v3" if entry["full_name"].startswith("query_") else "public" - domains.append({ + is_scalar = entry.get("scalar", False) + domain = { "name": f"{schema}.{entry['typname']}", "type": "jsonb", "variant": "", "base": "jsonb", - "shape": "stevec", - "capabilities": ["json"], + "capabilities": ["storage"] if is_scalar else ["json"], "supportedOperators": [], "termFunctions": _term_functions(entry.get("terms", [])), - }) + } + if not is_scalar: + domain["shape"] = "stevec" + domains.append(domain) domains.sort(key=lambda d: (d["type"], d["name"])) return domains diff --git a/tasks/docs/generate/xml-to-markdown.py b/tasks/docs/generate/xml-to-markdown.py index 76afc9281..d2bb46982 100755 --- a/tasks/docs/generate/xml-to-markdown.py +++ b/tasks/docs/generate/xml-to-markdown.py @@ -37,6 +37,24 @@ def generate_anchor(signature): anchor = anchor.strip('-') return anchor +def extract_code_text(element): + """Verbatim text of a , preserving significant whitespace. + + Doxygen encodes each space inside a code block as an empty element + and each source line as a , so the text has to be reassembled + rather than read off. No clean_text here — that is what the caller applies + once, folding the line breaks into a single readable line. + """ + parts = [] + if element.text: + parts.append(element.text) + for child in element: + parts.append(' ' if child.tag == 'sp' else extract_code_text(child)) + if child.tail: + parts.append(child.tail) + return ''.join(parts) + + def extract_para_text(element): """Extract text from para elements, including nested content""" if element is None: @@ -54,6 +72,12 @@ def extract_para_text(element): elif child.tag == 'computeroutput': if child.text: parts.append(child.text) + elif child.tag == 'programlisting': + # An @code block. Walk it verbatim rather than recursing: the + # generic path calls clean_text on every nested fragment, which + # strips the boundary spaces between runs and welds the + # code together ("SELECTeql_v3.grouped_value(...)"). + parts.append(extract_code_text(child)) else: parts.append(extract_para_text(child)) @@ -177,6 +201,31 @@ def extract_description(desc_element): return '\n\n'.join(lines) +def extract_aggregate_params(memberdef, param_docs): + """Parameters of a CREATE AGGREGATE, read from the declaration. + + Doxygen's C++ parse of an aggregate's argument list is unreliable — a + schema-qualified type splits into a bogus name/type pair, and a bare one is + dropped — but keeps the list verbatim ("(public.eql_v3_json)"). + Aggregate argument lists are positional and unnamed in SQL, so take the + types from there and the names/descriptions from the @param tags in order. + """ + argsstring = memberdef.find('argsstring') + text = (argsstring.text or '').strip() if argsstring is not None else '' + if text.startswith('(') and text.endswith(')'): + text = text[1:-1] + + params = [] + for i, arg_type in enumerate([a.strip() for a in text.split(',') if a.strip()]): + doc = param_docs[i] if i < len(param_docs) else None + params.append({ + 'name': doc['name'] if doc else '', + 'type': arg_type, + 'description': doc['description'] if doc else '', + }) + return params + + def process_function(memberdef): """Extract function documentation from memberdef element""" name = memberdef.find('name') @@ -230,6 +279,16 @@ def process_function(memberdef): type_text = extract_para_text(type_elem) if type_elem is not None else '' is_private = func_name.startswith('_') or 'eql_v3_internal' in type_text + # CREATE AGGREGATE has no C++ analogue. Once the Doxygen input filter has + # stripped the definition body the name and argument list survive, but the + # C++ parse still cannot tell a SQL argument TYPE from a parameter NAME + # (`(public.eql_v3_bigint_ord)` arrives as a parameter *named* `public.`), + # and it leaves "CREATE AGGREGATE " sitting in where a return + # type belongs. Both are unambiguous in the source, so for aggregates read + # the argument types off the declaration and the return type off @return + # rather than trusting the C++ parse. + is_aggregate = type_text.upper().startswith('CREATE AGGREGATE') + # Extract descriptions brief = extract_description(memberdef.find('briefdescription')) detailed_elem = memberdef.find('detaileddescription') @@ -290,6 +349,9 @@ def process_function(memberdef): } params.append(param_info) + if is_aggregate: + params = extract_aggregate_params(memberdef, param_docs) + # Extract simplesects (return, note, warning, see, etc.) simplesects = extract_simplesects(detailed_elem) @@ -302,8 +364,17 @@ def process_function(memberdef): return_type_text = '' if argsstring is not None and argsstring.text: - # Look for RETURNS keyword in argsstring - returns_match = re.search(r'RETURNS\s+([^\s]+)', argsstring.text) + # Look for RETURNS keyword in argsstring. + # + # Stop at "(" as well as whitespace. A set-returning `RETURNS TABLE (col + # type, ...)` reaches Doxygen as a C++ argument list, which it truncates + # at the first comma — `() RETURNS TABLE(severity text` — so matching to + # the next space published the fragment "TABLE(severity" as the return + # type. The column list is unrecoverable from the XML (joining the + # declaration onto one line does not help; the truncation is at the + # comma, not the line break), but it is spelled out in full by the + # @return tag that lands in `return_desc`. + returns_match = re.search(r'RETURNS\s+([^\s(]+)', argsstring.text) if returns_match: return_type_text = returns_match.group(1) # Debug: Check if already has backticks from XML @@ -313,6 +384,12 @@ def process_function(memberdef): # Debug print #print(f"DEBUG: Extracted from argsstring: {return_type_text}") + # An aggregate declares no RETURNS clause, and its holds + # "CREATE AGGREGATE " rather than a type. @return carries it. + if is_aggregate and not return_type_text: + return_desc = simplesects.get('return', '').strip() + return_type_text = return_desc.split()[0] if return_desc else '' + # Fallback to type element if not found in argsstring if not return_type_text: return_type = memberdef.find('type')