diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 68556daa..cbe72a1c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -133,7 +133,7 @@ All jobs run on `blacksmith-16vcpu-ubuntu-2204`. "PG set" follows the event | **schema** | `test:schema` | v2.2 / v2.3 payload JSON-schema validation | no | no | | **rust-crates** | `test:crates` + `types:check` | `cargo fmt --check`, clippy + `cargo test` for `eql-domains` / `eql-codegen` / `eql-tests-macros` / `eql-bindings`; verify TS bindings + JSON schemas are fresh | no | no | | **codegen** | `codegen:parity` | Regenerate encrypted-domain SQL in place + `git diff` drift gate (committed `src/v3/scalars/` matches the generator) | no | no | -| **self-contained-v3** | `test:self_contained_v3` | `eql_v3` surface has no `eql_v2` dependency | no | no | +| **self-contained-v3** | `test:self_contained_v3`, `test:installer_complete`, `test:symbol_order_v3`, `test:build_ordering_helpers` | `eql_v3` surface has no `eql_v2` dependency; installer contains every ordered file; singleton symbols defined before use (overloads are resolved exactly by the `clean-install` job's `test:clean_install_v3`) | no | no | | **matrix-coverage** | `test:matrix:inventory` (+`:jsonb_entry`, `:v3-jsonb`) + `test:matrix:catalog-coverage` | Scalar-matrix test-name snapshots are not silently dropped; catalog surface is covered | no | no | | **splinter** | `test:splinter` | Supabase/Splinter lints over the installed EQL | yes (PG17) | no | | **ci-required** | — | aggregator: every needed job is `success`/`skipped` | no | no | diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index cceea834..8667f476 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -409,6 +409,12 @@ jobs: run: mise run clean && mise run --force build - name: Assert eql_v3 is self-contained run: mise run test:self_contained_v3 + - name: Assert the installer contains every ordered file + run: mise run test:installer_complete + - name: Symbol-order cross-check (v3) + run: mise run test:symbol_order_v3 + - name: Build-ordering helper unit tests + run: mise run test:build_ordering_helpers matrix-coverage: name: "Matrix coverage inventory" diff --git a/.gitignore b/.gitignore index af3f1119..28d40fb2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,8 @@ deps-ordered.txt deps-supabase.txt deps-ordered-supabase.txt -src/deps-v3.txt src/deps-ordered-v3.txt +src/deps-ordered-v3.txt.tmp # Generated by tasks/build.sh from src/v3/version.template (eql_v3.version()). src/v3/version.sql diff --git a/CLAUDE.md b/CLAUDE.md index cc21f3b4..5a20b0db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,7 @@ This project uses `mise` for task management. Common commands: - `cipherstash-encrypt.sql` - The sole installer: the self-contained `eql_v3` surface, globbed from `src/v3` only (no `eql_v2`; installable into a DB with no `eql_v2` present) - `cipherstash-encrypt-uninstall.sql` - Matching uninstaller -There are no longer separate Main / Supabase / Protect / v3-only build variants. The combined `eql_v2` build that previously produced multiple artefacts has been removed; the v3 surface now ships as one self-contained installer under the canonical `cipherstash-encrypt.sql` name (`tasks/build.sh` globs `src/v3` only). Because the surface owns no `eql_v2` dependency, it is already Supabase / managed-Postgres compatible (functional indexes over extractors, no superuser-only operator classes) without a dedicated subset build. Self-containment — no `-- REQUIRE:` edge pointing outside `src/v3`, no `eql_v2.` anywhere in the surface — is enforced at build time by `verify_v3_self_contained` in `tasks/build.sh` and CI-gated by `mise run test:self_contained_v3`. +There are no longer separate Main / Supabase / Protect / v3-only build variants. The combined `eql_v2` build that previously produced multiple artefacts has been removed; the v3 surface now ships as one self-contained installer under the canonical `cipherstash-encrypt.sql` name (`eql-codegen order` walks `src/v3` only). Because the surface owns no `eql_v2` dependency, it is already Supabase / managed-Postgres compatible (functional indexes over extractors, no superuser-only operator classes) without a dedicated subset build. Self-containment — no `-- REQUIRE:` edge pointing outside `src/v3`, no `eql_v2.` anywhere in the surface — is enforced at build time by `surface_order`'s `OutsideSurface` error (`crates/eql-codegen/src/ordering.rs`) and CI-gated by `mise run test:self_contained_v3`. ## Project Architecture @@ -189,7 +189,7 @@ HTML output is also generated in `docs/api/html/` for local preview only. - SQL files are modular - put operator wrappers in `operators.sql`, implementation in `functions.sql` - All SQL files must have `-- REQUIRE:` dependency declarations -- Build system uses `tsort` to resolve dependency order +- Build system resolves dependency order with `cargo run -p eql-codegen -- order`, which walks the whole `src/v3` surface once and topologically sorts it from the `-- REQUIRE:` edges. Dangling targets, edges leaving `src/v3`, and cycles all fail the build. - **Documentation**: All functions/types must have Doxygen comments (see Documentation Standards above) ### Function Language Choice (SQL vs PL/pgSQL) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2b351ae0..910fea13 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -88,8 +88,7 @@ These are the important files and directories in the repo: │ │ │ └── / <-- e.g. integer/, text/, boolean/ (generated, committed in place) │ │ ├── jsonb/ <-- jsonb SteVec support │ │ └── lint/ <-- structural lints -│ ├── deps-v3.txt <-- REQUIRE edges for the v3 surface -│ ├── deps-ordered-v3.txt <-- tsorted build order +│ ├── deps-ordered-v3.txt <-- install order, emitted by `eql-codegen order` │ └── README.md ├── docs/ <-- reference, concept, and API documentation ├── tests/ <-- test framework and fixtures @@ -223,10 +222,14 @@ At minimum, a file references the schema: -- REQUIRE: src/v3/schema.sql ``` -The build collects these edges into `src/deps-v3.txt`, resolves them with -`tsort` into `src/deps-ordered-v3.txt`, and concatenates the files in -dependency order to produce a single installer. The build fails loudly if a -file referenced in the dependency list does not exist. +`cargo run -p eql-codegen -- order` walks the whole `src/v3` tree once, collects +these edges, and topologically sorts them into `src/deps-ordered-v3.txt`; the +build then concatenates the files in that order to produce a single installer. +Generated and hand-written files are ordered together, so nothing can fall +between two enumerations and be dropped. The build fails loudly if a `-- REQUIRE:` +target does not exist, if an edge leaves `src/v3`, or if the edges form a cycle. +`mise run test:installer_complete` then asserts the installer actually contains +every ordered file's body. The `eql_v3` surface is **self-contained**: no `eql_v2.` reference appears anywhere under `src/v3/`. This invariant is enforced in CI by diff --git a/crates/eql-codegen/src/generate.rs b/crates/eql-codegen/src/generate.rs index 2b66bd88..2b3bdef1 100644 --- a/crates/eql-codegen/src/generate.rs +++ b/crates/eql-codegen/src/generate.rs @@ -1087,6 +1087,9 @@ pub fn generate_all(out_root: &Path) -> Result { } } + // No ordering manifest is emitted here. The installer order is derived by + // `eql-codegen order` from a single walk of the whole src/v3 surface, so the + // generator has no say in — and cannot disagree with — what gets ordered. let names: Vec<&str> = eql_domains::families_with_scalar_domains() .map(|s| s.name) .collect(); diff --git a/crates/eql-codegen/src/lib.rs b/crates/eql-codegen/src/lib.rs index 1242e08d..f05c70bf 100644 --- a/crates/eql-codegen/src/lib.rs +++ b/crates/eql-codegen/src/lib.rs @@ -13,6 +13,7 @@ pub mod context; pub mod dump; pub mod generate; pub mod operator_surface; +pub mod ordering; pub mod writer; /// The repository root, derived from this crate's manifest dir (the generator diff --git a/crates/eql-codegen/src/main.rs b/crates/eql-codegen/src/main.rs index 8d200d68..1fc7eba0 100644 --- a/crates/eql-codegen/src/main.rs +++ b/crates/eql-codegen/src/main.rs @@ -69,6 +69,37 @@ fn main() -> ExitCode { } } + // `order`: print the install order of the whole src/v3 SQL surface, one + // repo-relative path per line, dependency before dependent. Consumed by + // tasks/build.sh (`> src/deps-ordered-v3.txt`), which concatenates the files + // in this order into release/cipherstash-encrypt.sql. + // + // The walk is the ONLY enumeration of the surface — hand-written and + // generated files are ordered together from their `-- REQUIRE:` edges, with + // no marker classifier and no separate codegen manifest to fall out of sync + // with. Missing REQUIRE targets, targets outside src/v3, and cycles are all + // hard errors here, so build.sh needs no post-hoc verification of the order. + if args.len() == 2 && args[1] == "order" { + let root = out_root(); + let result = eql_codegen::ordering::walk_v3_surface(&root) + .map_err(|e| format!("walking {}/src/v3: {e}", root.display())) + .and_then(|files| { + eql_codegen::ordering::surface_order(&files).map_err(|e| e.to_string()) + }); + match result { + Ok(order) => { + for path in &order { + println!("{path}"); + } + return ExitCode::SUCCESS; + } + Err(e) => { + eprintln!("error: {e}"); + return ExitCode::FAILURE; + } + } + } + // `clean`: remove the generated SQL surface (marker-aware) under every // src/v3/scalars/* type dir. Replaces build.sh's filename-pattern sweep; // hand-written files (no AUTO-GENERATED marker) are preserved. @@ -102,6 +133,7 @@ fn main() -> ExitCode { } eprintln!("Usage: eql-codegen (generate all types)"); + eprintln!(" eql-codegen order (print the src/v3 install order, one path per line)"); eprintln!(" eql-codegen clean (remove the generated SQL surface)"); eprintln!(" eql-codegen list-types (print catalog tokens)"); eprintln!(" eql-codegen list-schemas (print owned schemas, public first)"); diff --git a/crates/eql-codegen/src/ordering.rs b/crates/eql-codegen/src/ordering.rs new file mode 100644 index 00000000..7be34a44 --- /dev/null +++ b/crates/eql-codegen/src/ordering.rs @@ -0,0 +1,563 @@ +//! Deterministic topological ordering of the whole `src/v3` SQL surface. +//! +//! One enumeration orders every file: hand-written and generated alike are +//! walked off disk, their `-- REQUIRE:` edges parsed, and the result linearized +//! by [`surface_order`]. There is deliberately no generated/hand-written +//! classifier here. An earlier design split the surface into two blocks — a +//! shell glob that skipped the `-- AUTOMATICALLY GENERATED FILE.` marker, and a +//! codegen-emitted manifest of `render_type` output. Two enumerations means two +//! predicates, and a file matching neither — a generated one rendered outside +//! `render_type`, say — is silently dropped from the installer while the build +//! stays green. You order exactly the set you walk, so that class of bug is +//! unrepresentable. `install_order_contains_every_v3_sql_file` (parity tests) +//! pins it against an independent walk. + +use std::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; +use std::fs; +use std::io; +use std::path::Path; + +/// The surface root, relative to the repo root. Every node and every +/// `-- REQUIRE:` target must live under it — the eql_v3 installer is +/// self-contained and owns no edge pointing outside this tree. +pub const SURFACE_ROOT: &str = "src/v3"; + +/// A dependency cycle in the surface — the topo-sort could not linearize. +#[derive(Debug)] +pub struct CycleError { + /// The nodes that never reached in-degree 0 (participate in / are blocked by a cycle). + pub remaining: Vec, +} + +impl std::fmt::Display for CycleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Names the files, not their provenance: the sort draws no + // generated/hand-written distinction, and a cycle is as likely to run + // through a hand-authored `-- REQUIRE:` edge as a rendered one. + write!( + f, + "-- REQUIRE: dependency cycle, these files never linearize: {}", + self.remaining.join(", ") + ) + } +} +impl std::error::Error for CycleError {} + +/// A `-- REQUIRE:` target that is not a node in the surface, or points outside it. +#[derive(Debug)] +pub enum OrderError { + /// Targets naming a file that does not exist in the walked surface. Subsumes + /// the old `verify_deps_exist` shell gate, which only checked the converse + /// (every *listed* file exists on disk) and so never noticed a file on disk + /// that no block listed. + UnknownTargets(Vec<(String, String)>), + /// Targets outside `src/v3`. The v3 installer is self-contained: an edge to + /// (say) `src/v2/foo.sql` would pull non-v3 SQL into the artefact. Subsumes + /// the old `verify_v3_self_contained` shell gate. + OutsideSurface(Vec<(String, String)>), + /// Files that `-- REQUIRE:` themselves. Always a typo — and a damaging one, + /// because the line almost certainly meant to name a *different* file, so the + /// real edge is missing and the order can be silently wrong. + /// + /// The old shell build emitted a self-edge for every file on purpose (`echo + /// "$sql_file $sql_file"`), because `tsort` only prints tokens that appear in + /// some edge; tolerating them was load-bearing then. The walk enumerates every + /// node directly, so nothing emits self-edges now and the tolerance protects + /// nothing but the typo. + SelfEdge(Vec), + /// The edges do not linearize. + Cycle(CycleError), +} + +impl std::fmt::Display for OrderError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Every offender is listed, not just the first: a REQUIRE typo tends to + // come in batches (a renamed file breaks every dependent at once), and + // one-at-a-time diagnostics turn that into one build per typo. + match self { + Self::UnknownTargets(v) => { + writeln!(f, "-- REQUIRE: target does not exist:")?; + for (file, dep) in v { + writeln!(f, " {file} requires {dep}")?; + } + write!(f, "check the -- REQUIRE: directives above for typos") + } + Self::OutsideSurface(v) => { + writeln!(f, "-- REQUIRE: target outside {SURFACE_ROOT}:")?; + for (file, dep) in v { + writeln!(f, " {file} requires {dep}")?; + } + write!( + f, + "the eql_v3 surface must be self-contained — no edge may leave {SURFACE_ROOT}" + ) + } + Self::SelfEdge(v) => { + // NOT reported as a cycle. It is one in graph terms, but "dependency + // cycle" sends the reader hunting a loop between files that does not + // exist, when the fix is one line in one file. + writeln!(f, "-- REQUIRE: file requires itself:")?; + for file in v { + writeln!(f, " {file} requires {file}")?; + } + write!( + f, + "a file cannot depend on itself — this line likely meant to name another file, \ + in which case the real dependency is missing" + ) + } + Self::Cycle(e) => write!(f, "{e}"), + } + } +} +impl std::error::Error for OrderError {} + +/// Linearize the whole surface. `files` is `(repo-relative path, its REQUIRE +/// targets)` for EVERY `.sql` file in the surface. +/// +/// Unlike [`topo_order`], which tolerates edges to non-nodes and self-edges, this +/// validates first: every target must be a node, must live under [`SURFACE_ROOT`], +/// and must not be the requiring file itself. The first two gates ran in shell +/// before; keeping them here means the invariant travels with the sort rather than +/// with whoever remembers to call the checker. +pub fn surface_order(files: &[(String, Vec)]) -> Result, OrderError> { + let nodes: BTreeSet<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); + let prefix = format!("{SURFACE_ROOT}/"); + let (mut outside, mut unknown, mut self_edges) = (Vec::new(), Vec::new(), Vec::new()); + for (file, deps) in files { + for dep in deps { + if !dep.starts_with(&prefix) { + outside.push((file.clone(), dep.clone())); + } else if !nodes.contains(dep.as_str()) { + unknown.push((file.clone(), dep.clone())); + } else if dep == file { + self_edges.push(file.clone()); + } + } + } + // Outside-surface first: such a target is also "unknown" (it is not a node), + // and the self-containment breach is the more actionable diagnosis. + if !outside.is_empty() { + return Err(OrderError::OutsideSurface(outside)); + } + if !unknown.is_empty() { + return Err(OrderError::UnknownTargets(unknown)); + } + if !self_edges.is_empty() { + self_edges.dedup(); + return Err(OrderError::SelfEdge(self_edges)); + } + topo_order(files).map_err(OrderError::Cycle) +} + +/// Walk `/src/v3` for the surface: every `.sql` file paired with its +/// `-- REQUIRE:` targets, sorted by path. `*_test.sql` is excluded (it is not +/// part of the installer). +/// +/// Symlinked subdirectories are NOT followed — `file_type()` reports the link +/// itself, where `Path::is_dir()` would resolve it and could walk out of the +/// tree. Mirrors the orphan-sweep guard in `generate.rs`. +pub fn walk_v3_surface(root: &Path) -> io::Result)>> { + let mut files = Vec::new(); + let mut stack = vec![root.join(SURFACE_ROOT)]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + stack.push(path); + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + if !name.ends_with(".sql") || name.ends_with("_test.sql") { + continue; + } + let rel = path + .strip_prefix(root) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + // Name the file. A bare `?` here surfaces as "stream did not contain + // valid UTF-8" with no indication of which of ~244 files is at fault. + let body = fs::read_to_string(&path).map_err(|e| { + io::Error::new(e.kind(), format!("reading {}: {e}", path.display())) + })?; + files.push((rel, requires_of(&body))); + } + } + files.sort(); + Ok(files) +} + +/// Read back the anchored `-- REQUIRE:` targets from a SQL body — either a body +/// rendered in-process from the typed `requires` vec, or one read off disk. +pub fn requires_of(body: &str) -> Vec { + body.lines() + .filter_map(|l| l.trim_start().strip_prefix("-- REQUIRE:")) + .flat_map(|rest| rest.split_whitespace().map(str::to_string)) + .collect() +} + +/// Deterministic topological order of `files`, each `(repo-relative path, its +/// REQUIRE targets)`. Kahn's algorithm with a min-heap keyed by path string gives +/// name-sorted tie-breaking ⇒ byte-reproducible output. +/// +/// Edges whose target is not itself a key in `files` are ignored rather than +/// rejected. That tolerance is why this is not public: on the real surface a +/// non-node target means a typo'd or escaping `-- REQUIRE:`, and silently +/// ignoring it would drop the very check [`surface_order`] exists to make. Go +/// through [`surface_order`], which validates the targets before sorting. +pub(crate) fn topo_order(files: &[(String, Vec)]) -> Result, CycleError> { + let nodes: BTreeSet<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); + let mut indeg: BTreeMap<&str, usize> = nodes.iter().map(|n| (*n, 0usize)).collect(); + let mut dependents: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (p, reqs) in files { + for dep in reqs { + let dep = dep.as_str(); + if dep != p.as_str() && nodes.contains(dep) { + *indeg.get_mut(p.as_str()).unwrap() += 1; + dependents.entry(dep).or_default().push(p.as_str()); + } + } + } + let mut ready: BinaryHeap> = indeg + .iter() + .filter(|(_, d)| **d == 0) + .map(|(p, _)| Reverse(*p)) + .collect(); + let mut order: Vec = Vec::with_capacity(files.len()); + while let Some(Reverse(n)) = ready.pop() { + order.push(n.to_string()); + if let Some(deps) = dependents.get(n) { + // Push order does not matter: `ready` is a min-heap keyed by path, so + // it — not the insertion sequence — decides what comes out next. + for d in deps { + let e = indeg.get_mut(*d).unwrap(); + *e -= 1; + if *e == 0 { + ready.push(Reverse(*d)); + } + } + } + } + if order.len() != nodes.len() { + let done: BTreeSet<&str> = order.iter().map(|s| s.as_str()).collect(); + let remaining = nodes + .iter() + .filter(|n| !done.contains(**n)) + .map(|s| s.to_string()) + .collect(); + return Err(CycleError { remaining }); + } + Ok(order) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Two independent nodes must come out in byte (name) order — reproducible. + #[test] + fn topo_order_is_name_sorted_for_independent_nodes() { + let files = vec![("b.sql".to_string(), vec![]), ("a.sql".to_string(), vec![])]; + assert_eq!(topo_order(&files).unwrap(), vec!["a.sql", "b.sql"]); + } + + // A dependency edge (file requires dep) orders dep first. + #[test] + fn topo_order_respects_intra_generated_edges() { + let files = vec![ + ("ops.sql".to_string(), vec!["types.sql".to_string()]), + ("types.sql".to_string(), vec![]), + ( + "agg.sql".to_string(), + vec!["ops.sql".to_string(), "types.sql".to_string()], + ), + ]; + let out = topo_order(&files).unwrap(); + let pos = |n: &str| out.iter().position(|x| x == n).unwrap(); + assert!(pos("types.sql") < pos("ops.sql")); + assert!(pos("ops.sql") < pos("agg.sql")); + } + + // Edges to files NOT in the set (hand-written prerequisites) are ignored: + // they never block ordering and never appear in the output. + #[test] + fn topo_order_ignores_external_edges() { + let files = vec![("t.sql".to_string(), vec!["src/v3/schema.sql".to_string()])]; + assert_eq!(topo_order(&files).unwrap(), vec!["t.sql"]); + } + + // Identical input twice ⇒ identical output (determinism invariant). + #[test] + fn topo_order_is_deterministic() { + let files = vec![ + ("c.sql".to_string(), vec!["a.sql".to_string()]), + ("a.sql".to_string(), vec![]), + ("b.sql".to_string(), vec!["a.sql".to_string()]), + ]; + assert_eq!(topo_order(&files).unwrap(), topo_order(&files).unwrap()); + // a first, then b, c by name. + assert_eq!(topo_order(&files).unwrap(), vec!["a.sql", "b.sql", "c.sql"]); + } + + // A cycle is a hard error naming the stuck nodes. + #[test] + fn topo_order_detects_cycle() { + let files = vec![ + ("a.sql".to_string(), vec!["b.sql".to_string()]), + ("b.sql".to_string(), vec!["a.sql".to_string()]), + ]; + let err = topo_order(&files).unwrap_err(); + assert!(err.remaining.contains(&"a.sql".to_string())); + assert!(err.remaining.contains(&"b.sql".to_string())); + } + + fn f(path: &str, deps: &[&str]) -> (String, Vec) { + ( + path.to_string(), + deps.iter().map(|d| d.to_string()).collect(), + ) + } + + // The happy path: every target is a node under src/v3, and edges are honoured. + #[test] + fn surface_order_linearizes_a_valid_surface() { + let files = vec![ + f("src/v3/ops.sql", &["src/v3/types.sql"]), + f("src/v3/types.sql", &["src/v3/schema.sql"]), + f("src/v3/schema.sql", &[]), + f("src/v3/orphan.sql", &[]), // no edges: must still be emitted + ]; + let out = surface_order(&files).unwrap(); + let pos = |n: &str| out.iter().position(|x| x == n).unwrap(); + assert!(pos("src/v3/schema.sql") < pos("src/v3/types.sql")); + assert!(pos("src/v3/types.sql") < pos("src/v3/ops.sql")); + assert!(out.contains(&"src/v3/orphan.sql".to_string())); + assert_eq!(out.len(), 4); + } + + // A REQUIRE naming a file that is not in the surface is a hard error. This is + // the gate the old shell `verify_deps_exist` could not express. + #[test] + fn surface_order_rejects_unknown_target() { + let files = vec![f("src/v3/a.sql", &["src/v3/missing.sql"])]; + let err = surface_order(&files).unwrap_err(); + let OrderError::UnknownTargets(v) = &err else { + panic!("expected UnknownTargets, got {err:?}"); + }; + assert_eq!( + v.as_slice(), + &[("src/v3/a.sql".into(), "src/v3/missing.sql".into())] + ); + assert!(err.to_string().contains("src/v3/missing.sql")); + } + + // An edge leaving src/v3 breaks self-containment, and is reported as such + // rather than as a generic unknown target. + #[test] + fn surface_order_rejects_target_outside_the_surface() { + let files = vec![f("src/v3/a.sql", &["src/v2/x.sql"]), f("src/v2/x.sql", &[])]; + let err = surface_order(&files).unwrap_err(); + assert!( + matches!(err, OrderError::OutsideSurface(_)), + "expected OutsideSurface, got {err:?}" + ); + assert!(err.to_string().contains("self-contained")); + } + + // `src/v3xyz/` must not pass the prefix check on a bare `starts_with("src/v3")`. + #[test] + fn surface_order_prefix_check_is_path_segment_exact() { + let files = vec![f("src/v3/a.sql", &["src/v3suffix/x.sql"])]; + assert!(matches!( + surface_order(&files).unwrap_err(), + OrderError::OutsideSurface(_) + )); + } + + // The cycle diagnostic must describe the surface it actually sorts. It once + // said "among generated files", inherited from the two-block design this + // module replaced — which sent a reader chasing the codegen when the fix was + // a `-- REQUIRE:` line in their own hand-written file. + #[test] + fn cycle_error_names_the_stuck_files_not_a_generated_block() { + let files = vec![ + f("src/v3/jsonb/a.sql", &["src/v3/jsonb/b.sql"]), + f("src/v3/jsonb/b.sql", &["src/v3/jsonb/a.sql"]), + ]; + let msg = surface_order(&files).unwrap_err().to_string(); + assert!( + !msg.contains("generated"), + "the sort has no generated/hand-written distinction; the message must not imply one: {msg}" + ); + assert!( + msg.contains("src/v3/jsonb/a.sql") && msg.contains("src/v3/jsonb/b.sql"), + "the message must name the stuck files: {msg}" + ); + } + + // A file that requires ITSELF is rejected, and NOT as a cycle. + // + // This tolerance used to be deliberate: the old shell build emitted a self-edge + // for every file (`echo "$sql_file $sql_file"`) because `tsort` only prints + // tokens appearing in some edge. That rationale died with the shell build — the + // walk enumerates nodes directly, so nothing emits self-edges now and the only + // way one appears is a hand-typo. A typo'd edge is not harmless: the line meant + // to name another file, so the real dependency is missing and the order can be + // silently wrong. + // + // Reported as SelfEdge, not Cycle: "dependency cycle" would send the reader + // hunting a loop between files when the fix is one line in one file. + #[test] + fn surface_order_rejects_a_self_edge() { + let files = vec![f("src/v3/a.sql", &["src/v3/a.sql"])]; + let err = surface_order(&files).unwrap_err(); + let OrderError::SelfEdge(v) = &err else { + panic!("expected SelfEdge, got {err:?}"); + }; + assert_eq!(v.as_slice(), &["src/v3/a.sql".to_string()]); + let msg = err.to_string(); + assert!( + msg.contains("requires itself") && !msg.contains("cycle"), + "a self-require must not be reported as a cycle: {msg}" + ); + } + + // A self-edge is rejected even when the file has other, legitimate edges — the + // typo hides among real REQUIRE lines, which is exactly how it would ship. + #[test] + fn surface_order_rejects_a_self_edge_among_valid_edges() { + let files = vec![ + f("src/v3/schema.sql", &[]), + f("src/v3/a.sql", &["src/v3/schema.sql", "src/v3/a.sql"]), + ]; + assert!(matches!( + surface_order(&files).unwrap_err(), + OrderError::SelfEdge(_) + )); + } + + // A cycle surfaces as a cycle, not as a silently truncated order. + #[test] + fn surface_order_propagates_cycles() { + let files = vec![ + f("src/v3/a.sql", &["src/v3/b.sql"]), + f("src/v3/b.sql", &["src/v3/a.sql"]), + ]; + assert!(matches!( + surface_order(&files).unwrap_err(), + OrderError::Cycle(_) + )); + } + + // Identical input twice => identical output. The build's byte-reproducibility + // rests on this (the monolith is concatenated in this order). + #[test] + fn surface_order_is_deterministic() { + let files = vec![ + f("src/v3/c.sql", &["src/v3/a.sql"]), + f("src/v3/a.sql", &[]), + f("src/v3/b.sql", &["src/v3/a.sql"]), + ]; + assert_eq!( + surface_order(&files).unwrap(), + surface_order(&files).unwrap() + ); + assert_eq!( + surface_order(&files).unwrap(), + vec!["src/v3/a.sql", "src/v3/b.sql", "src/v3/c.sql"] + ); + } + + // The walk finds nested files, reads their edges, skips `*_test.sql`, and is + // blind to the generated/hand-written marker (both kinds are ordered together). + #[test] + fn walk_v3_surface_collects_every_sql_file_with_its_edges() { + let d = crate::writer::test_support::tempdir(); + let v3 = d.path().join("src/v3"); + fs::create_dir_all(v3.join("scalars/integer")).unwrap(); + fs::write(v3.join("schema.sql"), "CREATE SCHEMA eql_v3;\n").unwrap(); + fs::write( + v3.join("scalars/cross_family.sql"), + "-- AUTOMATICALLY GENERATED FILE.\n-- REQUIRE: src/v3/schema.sql\n", + ) + .unwrap(); + fs::write( + v3.join("scalars/integer/integer_types.sql"), + "-- REQUIRE: src/v3/schema.sql\n", + ) + .unwrap(); + fs::write(v3.join("scalars/integer/x_test.sql"), "SELECT 1;\n").unwrap(); + fs::write(v3.join("notes.md"), "not sql\n").unwrap(); + + let files = walk_v3_surface(d.path()).unwrap(); + let paths: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); + // Path-sorted, so `scalars/` precedes `schema.sql`. The walk order carries + // no dependency meaning — surface_order supplies that, below. + assert_eq!( + paths, + vec![ + "src/v3/scalars/cross_family.sql", + "src/v3/scalars/integer/integer_types.sql", + "src/v3/schema.sql", + ], + "walk must be sorted, skip *_test.sql and non-sql, and include the \ + cross-family generated file a two-block build would drop" + ); + // The cross-family file's edges came back with it, not just its path. + assert_eq!(files[0].1, vec!["src/v3/schema.sql"]); + // And the walked surface linearizes: schema.sql moves ahead of its dependents. + assert_eq!(surface_order(&files).unwrap()[0], "src/v3/schema.sql"); + } + + // A symlinked subdirectory is not followed: `file_type()` reports the link, + // where `Path::is_dir()` would resolve it and walk outside the tree. + #[cfg(unix)] + #[test] + fn walk_v3_surface_does_not_follow_symlinked_subdir() { + let d = crate::writer::test_support::tempdir(); + let v3 = d.path().join("src/v3"); + fs::create_dir_all(&v3).unwrap(); + fs::write(v3.join("schema.sql"), "SELECT 1;\n").unwrap(); + let outside = d.path().join("outside"); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("stray.sql"), "SELECT 2;\n").unwrap(); + std::os::unix::fs::symlink(&outside, v3.join("linked")).unwrap(); + + let files = walk_v3_surface(d.path()).unwrap(); + let paths: Vec<&str> = files.iter().map(|(p, _)| p.as_str()).collect(); + assert_eq!(paths, vec!["src/v3/schema.sql"]); + } + + // An unreadable file names itself. `fs::read_to_string`'s own error is + // "stream did not contain valid UTF-8" — true, and useless across ~244 files. + #[test] + fn walk_v3_surface_names_the_file_it_could_not_read() { + let d = crate::writer::test_support::tempdir(); + let v3 = d.path().join("src/v3"); + fs::create_dir_all(&v3).unwrap(); + fs::write(v3.join("bad.sql"), [0xff, 0xfe, 0x00]).unwrap(); + + let err = walk_v3_surface(d.path()).unwrap_err(); + assert!( + err.to_string().contains("bad.sql"), + "the error must name the offending file, got: {err}" + ); + } + + // requires_of reads back anchored `-- REQUIRE:` lines from a rendered body. + #[test] + fn requires_of_reads_anchored_directives() { + let body = "-- AUTOMATICALLY GENERATED FILE.\n-- REQUIRE: src/v3/schema.sql\n-- REQUIRE: a.sql b.sql\nSELECT 1; -- REQUIRE in prose\n"; + assert_eq!( + requires_of(body), + vec![ + "src/v3/schema.sql".to_string(), + "a.sql".to_string(), + "b.sql".to_string() + ] + ); + } +} diff --git a/crates/eql-codegen/tests/cli.rs b/crates/eql-codegen/tests/cli.rs index 3a87548d..4beced97 100644 --- a/crates/eql-codegen/tests/cli.rs +++ b/crates/eql-codegen/tests/cli.rs @@ -86,6 +86,72 @@ fn list_schemas_subcommand_prints_owned_schemas() { ); } +/// `order` exits 0 and prints the real surface's install order, one repo-relative +/// path per line, dependency first. `tasks/build.sh` redirects this straight into +/// `src/deps-ordered-v3.txt` and concatenates the files in this order, so the +/// stdout contract is pinned here: nothing but paths (no banner, no progress). +#[test] +fn order_subcommand_prints_the_install_order() { + let out = Command::new(bin()) + .arg("order") + .output() + .expect("run eql-codegen order"); + assert!( + out.status.success(), + "order should exit 0; stderr: {}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!( + lines.first(), + Some(&"src/v3/schema.sql"), + "schema.sql creates the schemas everything else requires, so it must come first" + ); + assert!( + lines + .iter() + .all(|l| l.starts_with("src/v3/") && l.ends_with(".sql")), + "stdout must be paths only — build.sh feeds it to `strip_require_lines` unfiltered" + ); +} + +/// A `-- REQUIRE:` naming a file that does not exist fails the build loudly +/// instead of emitting a short order. The two-block scheme this replaced could +/// omit a file and still exit 0 — a green build with the file missing from +/// `release/cipherstash-encrypt.sql`. +#[test] +fn order_subcommand_fails_on_a_dangling_require() { + let root = tempdir(); + let v3 = root.0.join("src/v3"); + std::fs::create_dir_all(&v3).unwrap(); + std::fs::write(v3.join("schema.sql"), "CREATE SCHEMA eql_v3;\n").unwrap(); + std::fs::write( + v3.join("broken.sql"), + "-- REQUIRE: src/v3/typo.sql\nSELECT 1;\n", + ) + .unwrap(); + + let out = Command::new(bin()) + .arg("order") + .env("EQL_CODEGEN_OUT_ROOT", root.0.as_os_str()) + .output() + .expect("run eql-codegen order"); + assert!( + !out.status.success(), + "a dangling REQUIRE must fail the build, not emit a partial order" + ); + assert!( + out.stdout.is_empty(), + "a failed order must print no partial order to stdout" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("src/v3/typo.sql") && stderr.contains("src/v3/broken.sql"), + "the error must name both the dangling target and the file requiring it, got:\n{stderr}" + ); +} + /// An unrecognised argument prints usage and exits 2 (the `ExitCode::from(2)` /// fall-through in `main.rs`). #[test] diff --git a/crates/eql-codegen/tests/parity.rs b/crates/eql-codegen/tests/parity.rs index e3fb90a0..d9ba18da 100644 --- a/crates/eql-codegen/tests/parity.rs +++ b/crates/eql-codegen/tests/parity.rs @@ -115,6 +115,76 @@ fn every_generated_sql_file_starts_with_marker() { ); } +/// Every `.sql` file in the real `src/v3` tree is in the install order — the +/// completeness invariant, checked against an independent walk rather than a +/// hardcoded count. +/// +/// This is the gate that a two-block build could not express. When the surface +/// was ordered as "hand-written files (globbed, minus the AUTO-GENERATED marker)" +/// plus "generated files (from a codegen manifest of `render_type` output)", a +/// cross-family generated file — marker-bearing, but rendered outside +/// `render_type` — matched neither predicate and would be silently dropped from +/// the installer. Set equality, not a count: a new cross-family generated file is +/// required here the moment it lands on disk. +/// +/// Runs against the real tree, not a tempdir: `surface_order` validates that +/// every `-- REQUIRE:` target is a node, and a generate-only tempdir has no +/// `schema.sql` or `sem/**` for the generated files to point at. +#[test] +fn install_order_contains_every_v3_sql_file() { + let root = repo_root(); + + // Independent of walk_v3_surface: if the walker under-collects, this diverges. + let mut on_disk: BTreeSet = BTreeSet::new(); + let mut stack = vec![root.join("src/v3")]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.push(path); + continue; + } + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".sql") && !name.ends_with("_test.sql") { + on_disk.insert( + path.strip_prefix(&root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"), + ); + } + } + } + + let files = eql_codegen::ordering::walk_v3_surface(&root).expect("walk src/v3"); + let order = eql_codegen::ordering::surface_order(&files).expect( + "src/v3 surface must linearize: every REQUIRE target a node under src/v3, no cycles", + ); + + // Check for duplicates BEFORE collapsing into a set, which would absorb them. + // `tasks/build.sh` concatenates the order line by line with no `uniq`, so a + // repeated path emits that file's DDL twice into the installer. Kahn's + // algorithm cannot produce one today; this pins that it stays that way. + let ordered: BTreeSet = order.iter().cloned().collect(); + assert_eq!( + order.len(), + ordered.len(), + "the install order contains a duplicate path — build.sh would emit its DDL twice" + ); + + assert_eq!( + ordered, on_disk, + "the install order must contain exactly the src/v3 SQL files on disk — a file \ + present on disk but absent from the order is silently missing from \ + release/cipherstash-encrypt.sql" + ); + assert!( + on_disk.contains("src/v3/schema.sql"), + "sanity: the walk found no schema.sql, so it is not seeing the real tree" + ); +} + #[test] fn generate_all_skips_non_scalar_families() { let tmp = tempdir("skip-non-scalar"); diff --git a/docs/reference/adding-a-scalar-encrypted-domain-type.md b/docs/reference/adding-a-scalar-encrypted-domain-type.md index dfc5952a..d59440f0 100644 --- a/docs/reference/adding-a-scalar-encrypted-domain-type.md +++ b/docs/reference/adding-a-scalar-encrypted-domain-type.md @@ -56,6 +56,17 @@ To add a scalar type `` (e.g. `bigint`), with Rust type `` (e.g. `i64`): no per-type codegen task. The generated `*_{types,functions,operators,aggregates}.sql` are committed in place under `src/v3/scalars//` — regenerate and commit the SQL diff alongside the catalog change. + - **Ordering is resolved by the codegen — you do nothing.** `eql-codegen order` + walks the whole `src/v3` surface once and topologically sorts it from the + `-- REQUIRE:` edges every file declares (name-sorted tie-break, cycle- and + dangling-target-checked); `tasks/build.sh` concatenates the files in that + order. Generated and hand-written files are ordered together — there is no + separate generated block, so a generated file cannot fall between the two. + Adding a catalog row needs no `-- REQUIRE:` edits to any generated file: the + renderers emit each file's edges. Only hand-written files under `src/v3/` + (SEM types, `jsonb/`, `schema.sql`, `crypto.sql`, `common.sql`, + `scalars/functions.sql`, `lint/lints.sql`, `*_extensions.sql`) carry + authored `-- REQUIRE:` edges. - The catalog row ALSO drives the **Rust payload bindings**: `eql-codegen bindings` (run first by `mise run types:generate`) regenerates the committed `crates/eql-bindings/src/v3/.rs` struct + `DomainType` diff --git a/mise.toml b/mise.toml index 844e913b..1c9d7ef8 100644 --- a/mise.toml +++ b/mise.toml @@ -225,6 +225,26 @@ run = """ cargo test -p eql-domains -p eql-codegen """ +[tasks."test:symbol_order_v3"] +description = "Cross-check installer symbol definition order (DB-free)" +depends = ["build"] +dir = "{{config_root}}" +run = """ +bash tasks/test/symbol_order_selftest.sh +bash tasks/test/verify_symbol_order_v3.sh src/deps-ordered-v3.txt +""" + +[tasks."test:installer_complete"] +description = "Assert the installer contains every ordered src/v3 file's body (DB-free)" +depends = ["build"] +dir = "{{config_root}}" +run = "bash tasks/test/verify_installer_complete.sh src/deps-ordered-v3.txt release/cipherstash-encrypt.sql" + +[tasks."test:build_ordering_helpers"] +description = "Unit tests for tasks/build/ordering.sh strip_require_lines (DB-free)" +dir = "{{config_root}}" +run = "bash tasks/test/build_ordering_helpers_test.sh" + [tasks."test:crates"] description = "Compile, lint and test the std-only Rust workspace crates (no database)" dir = "{{config_root}}" diff --git a/tasks/build.sh b/tasks/build.sh index 3b362560..cf8cd28d 100755 --- a/tasks/build.sh +++ b/tasks/build.sh @@ -1,18 +1,34 @@ #!/usr/bin/env bash #MISE description="Build SQL into single release file" #MISE alias="b" -#MISE sources=["src/v3/**/*.sql", "src/v3/version.template", "tasks/pin_search_path_v3.sql", "tasks/uninstall-v3.sql", "crates/eql-domains/src/**/*.rs", "crates/eql-codegen/src/**/*.rs"] -#MISE outputs=["release/cipherstash-encrypt.sql","release/cipherstash-encrypt-uninstall.sql"] +#MISE sources=["src/v3/**/*.sql", "src/v3/version.template", "tasks/pin_search_path_v3.sql", "tasks/uninstall-v3.sql", "crates/eql-domains/src/**/*.rs", "crates/eql-codegen/src/**/*.rs", "Cargo.toml", "Cargo.lock", "crates/eql-codegen/Cargo.toml", "crates/eql-domains/Cargo.toml", "tasks/build/ordering.sh", "tasks/test/verify_symbol_order_v3.sh", "tasks/test/verify_installer_complete.sh", "tasks/test/symbol_order_allowlist.txt"] +#MISE outputs=["release/cipherstash-encrypt.sql","release/cipherstash-encrypt-uninstall.sql","src/deps-ordered-v3.txt"] #USAGE flag "--version " help="Specify release version of EQL" default="DEV" #!/bin/bash set -euo pipefail +# ordering.sh shapes the installer (strip_require_lines), and the two verify +# scripts below gate it. All four are in #MISE sources: a cache hit skips this +# script entirely, gates included, so an edit to any of them must invalidate the +# build rather than re-serve an installer built by the old logic. +# +# The Cargo manifests and Cargo.lock are sources for the same reason: the two +# `cargo run` steps below render this artefact, and eql-codegen's own deps decide +# what they render — minijinja templates the SQL, prettyplease (=0.2.37) formats +# the bindings. A dep bump changes the output with no .rs file touched. +source tasks/build/ordering.sh + +# A failed `eql-codegen order` leaves its temp behind; don't strand it. +trap 'rm -f src/deps-ordered-v3.txt.tmp' EXIT + # Regenerate encrypted-domain SQL from the Rust catalog before building. -# Generated files (src/v3/scalars//_*.sql) are gitignored; the -# catalog at crates/eql-domains/src (eql-domains::CATALOG) is the source of -# truth, rendered by the eql-codegen binary. +# The generated files (src/v3/scalars//_*.sql) are COMMITTED in place and +# drift-gated by `mise run codegen:parity`; only src/v3/version.sql and the +# src/deps-ordered-v3.txt build intermediate are gitignored. The catalog at +# crates/eql-domains/src (eql-domains::CATALOG) is the source of truth, rendered +# by the eql-codegen binary. # # eql-codegen owns orphan removal: it writes every current file first (each via # an atomic temp+rename), then prunes stale generated SQL across ALL @@ -25,96 +41,60 @@ set -euo pipefail # # The plaintext fixture lists are not generated — the SQLx tests read them # straight from the catalog (eql_domains::INT4_VALUES / …). -cargo run -p eql-codegen - -# Fail loudly if any file referenced in a tsorted dep list doesn't exist. -# Without this, `xargs cat` would print `cat: foo.sql: No such file or directory` -# and continue — silently producing an incomplete release artefact. -verify_deps_exist() { - local dep_file=$1 - local missing=0 - while IFS= read -r f; do - if [[ ! -f "$f" ]]; then - echo "ERROR: $dep_file references missing file: $f" >&2 - missing=1 - fi - done < "$dep_file" - if [[ $missing -ne 0 ]]; then - echo "ERROR: dependency graph references missing files (see above). Check -- REQUIRE: directives." >&2 - exit 1 - fi -} - -# Fail loudly if any v3 REQUIRE edge points OUTSIDE src/v3. The v3-only build -# must be self-contained (no eql_v2 coupling); a stray `-- REQUIRE: src/...` -# edge to a non-v3 file would silently pull eql_v2 SQL into the v3 artefact (or -# tsort would drop it), breaking self-containment. Each line in deps-v3.txt is -# " "; self-edges (file == dep) are skipped, every other dep target -# must start with src/v3/. -verify_v3_self_contained() { - local dep_file=$1 - local offending=0 - while IFS=' ' read -r src dep; do - [[ -z "$dep" ]] && continue - [[ "$src" == "$dep" ]] && continue - if [[ "$dep" != src/v3/* ]]; then - echo "ERROR: v3 REQUIRE edge points outside src/v3: $src -- REQUIRE: $dep" >&2 - offending=1 - fi - done < "$dep_file" - if [[ $offending -ne 0 ]]; then - echo "ERROR: v3-only build is not self-contained — a -- REQUIRE: target lives outside src/v3 (see above)." >&2 - exit 1 - fi -} +cargo run -q -p eql-codegen mkdir -p release rm -f release/cipherstash-encrypt.sql rm -f release/cipherstash-encrypt-uninstall.sql - -rm -f src/deps-v3.txt -rm -f src/deps-ordered-v3.txt +rm -f src/deps-ordered-v3.txt src/deps-ordered-v3.txt.tmp rm -f src/v3/version.sql -# Bake the release version into eql_v3.version() (and the eql_v3 schema -# comment) before the glob below picks it up. The version is supplied via -# `mise run build --version ` (the `usage_version` env var mise derives -# from the #USAGE flag); local builds with no flag fall back to DEV. The -# generated src/v3/version.sql is gitignored, like the other generated v3 SQL. +# Bake the release version into eql_v3.version() (and the eql_v3 schema comment). +# The version is supplied via `mise run build --version ` (the +# `usage_version` env var mise derives from the #USAGE flag); local builds with +# no flag fall back to DEV. The generated src/v3/version.sql is gitignored. +# +# This MUST precede `eql-codegen order` below: the ordering walks the surface on +# disk, so version.sql has to exist to be ordered into the installer. RELEASE_VERSION=${usage_version:-DEV} sed "s/\$RELEASE_VERSION/$RELEASE_VERSION/g" src/v3/version.template > src/v3/version.sql -# The self-contained eql_v3 surface — schema, SEM types, scalar domains — -# globbed from src/v3 ONLY. This is the sole EQL artifact: it owns no eql_v2 -# dependency (CI-gated by verify_v3_self_contained below + test:self_contained_v3), -# and it is written under the canonical release name now that the combined v2 -# build that previously produced that name is gone. -find src/v3 -type f -path "*.sql" ! -path "*_test.sql" | while IFS= read -r sql_file; do - echo "$sql_file" - - echo "$sql_file $sql_file" >> src/deps-v3.txt - - while IFS= read -r line; do - if [[ "$line" == *"-- REQUIRE:"* ]]; then - deps=${line#*-- REQUIRE: } - for dep in $deps; do - echo "$sql_file $dep" >> src/deps-v3.txt - done - fi - done < "$sql_file" -done - -verify_v3_self_contained src/deps-v3.txt +# Resolve the install order of the whole eql_v3 surface — schema, SEM types, +# hand-written jsonb, generated scalars, version.sql — in ONE walk of src/v3, +# topologically sorted from the `-- REQUIRE:` edges every file declares. +# +# `eql-codegen order` is the sole enumeration of the surface, and it fails the +# build on a missing REQUIRE target, on an edge leaving src/v3 (self-containment, +# also gated by test:self_contained_v3), and on a dependency cycle. It replaces a +# two-block scheme — shell-globbed hand-written files, plus a codegen manifest of +# the generated ones — whose two enumerations could disagree about a file and +# silently drop it from the installer. Ordering what you walk makes that +# unrepresentable; `install_order_contains_every_v3_sql_file` in the codegen +# crate's parity tests pins the invariant. +# +# Written via a temp file so an aborted order leaves no truncated list behind for +# the downstream tasks (test:self_contained_v3, test:symbol_order_v3) to read. +cargo run -q -p eql-codegen -- order > src/deps-ordered-v3.txt.tmp +mv src/deps-ordered-v3.txt.tmp src/deps-ordered-v3.txt -cat src/deps-v3.txt | tsort | tac > src/deps-ordered-v3.txt -verify_deps_exist src/deps-ordered-v3.txt +bash tasks/test/verify_symbol_order_v3.sh src/deps-ordered-v3.txt -cat src/deps-ordered-v3.txt | xargs cat | grep -v REQUIRE >> release/cipherstash-encrypt.sql +: > release/cipherstash-encrypt.sql +while IFS= read -r f; do + strip_require_lines "$f" >> release/cipherstash-encrypt.sql +done < src/deps-ordered-v3.txt cat tasks/pin_search_path_v3.sql >> release/cipherstash-encrypt.sql +# `eql-codegen order` guarantees the ORDER contains every file on disk. This gate +# closes the layer below — that the concat loop above actually emitted each ordered +# file's body. 93 of the ~244 v3 files are leaves (required by nothing, defining +# nothing another file references), so dropping one yields an installer that applies +# cleanly and passes the symbol checker while silently shipping less than it should. +bash tasks/test/verify_installer_complete.sh src/deps-ordered-v3.txt release/cipherstash-encrypt.sql + cat tasks/uninstall-v3.sql >> release/cipherstash-encrypt-uninstall.sql diff --git a/tasks/build/ordering.sh b/tasks/build/ordering.sh new file mode 100644 index 00000000..99d5b27c --- /dev/null +++ b/tasks/build/ordering.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Sourceable helpers for the eql_v3 build. No side effects on source; each +# function is pure w.r.t. its args. Shared with the staged-installer refactor +# (do not fork strip_require_lines). +# +# Dependency ordering itself now lives in `eql-codegen order` (see +# crates/eql-codegen/src/ordering.rs), which walks src/v3 once and topologically +# sorts the whole surface. The shell tsort wrapper and linearization checker this +# file used to carry are gone with the two-block build they served. + +# Emit a file's body with anchored `-- REQUIRE:` directive lines removed. Anchored +# (allows leading whitespace) so a body line that merely contains the substring +# "REQUIRE" survives — unlike the old unanchored `grep -v REQUIRE`. +strip_require_lines() { + local rc=0 + grep -vE '^[[:space:]]*-- REQUIRE:' "$1" || rc=$? + # grep exits 1 when EVERY line matched the exclude (nothing left) — not an + # error. Exit codes >= 2 (missing file, unreadable, bad regex) are real + # failures: propagate so `set -e` aborts assembly instead of silently emitting + # a truncated monolith. (The blanket `|| true` this replaces hid exit 2.) + (( rc <= 1 )) +} diff --git a/tasks/test/build_ordering_helpers_test.sh b/tasks/test/build_ordering_helpers_test.sh new file mode 100755 index 00000000..bcc7324e --- /dev/null +++ b/tasks/test/build_ordering_helpers_test.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +#MISE description="DB-free unit tests for tasks/build/ordering.sh (anchored REQUIRE strip)" +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" +source tasks/build/ordering.sh + +# strip_require_lines is all that remains in ordering.sh: the tsort cycle gate and +# the linearization checker moved into `eql-codegen order`, and are covered by the +# eql-codegen crate's ordering:: unit tests. + +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT + +# 1. Anchored strip keeps a body line that merely contains the substring REQUIRE. +printf -- '-- REQUIRE: src/v3/schema.sql\nSELECT 1; -- the REQUIRE keyword in prose\n' > "$tmp/body.sql" +out="$(strip_require_lines "$tmp/body.sql")" +[[ "$out" == "SELECT 1; -- the REQUIRE keyword in prose" ]] || { echo "FAIL: strip removed non-directive line: [$out]"; exit 1; } +echo "ok: anchored strip preserves non-directive REQUIRE substring" + +# 2. strip_require_lines must FAIL (not silently succeed) on an unreadable file — +# a real grep error (exit >= 2) propagates so `set -e` aborts a truncated build. +if strip_require_lines "$tmp/does-not-exist.sql" 2>/dev/null; then + echo "FAIL: strip_require_lines swallowed a missing-file error"; exit 1 +fi +echo "ok: strip_require_lines propagates a real grep failure" + +# 3. A file that is ENTIRELY -- REQUIRE: lines (grep exit 1, nothing left) is +# NOT an error — it contributes no body and must succeed with empty output. +printf -- '-- REQUIRE: a\n-- REQUIRE: b\n' > "$tmp/allreq.sql" +out="$(strip_require_lines "$tmp/allreq.sql")" || { echo "FAIL: all-REQUIRE file treated as error"; exit 1; } +[[ -z "$out" ]] || { echo "FAIL: all-REQUIRE file produced output: [$out]"; exit 1; } +echo "ok: all-REQUIRE file succeeds with empty output" + +echo "ALL build-ordering helper tests passed" diff --git a/tasks/test/self_contained_v3.sh b/tasks/test/self_contained_v3.sh index b9fa44c8..74acf907 100755 --- a/tasks/test/self_contained_v3.sh +++ b/tasks/test/self_contained_v3.sh @@ -21,7 +21,11 @@ if grep -rnE 'eql_v2[._]' src/v3; then fi # File level (design goal 2): the v3-only dependency closure pulls in no file -# outside src/v3/. tsort output is one path per line. +# outside src/v3/. `eql-codegen order` emits one repo-relative path per line. +# +# Belt-and-braces: surface_order already rejects any `-- REQUIRE:` edge leaving +# src/v3, and the walk is rooted at src/v3, so every node is under it by +# construction. This gate would only fire again if that root ever widened. if [[ ! -f src/deps-ordered-v3.txt ]]; then echo "ERROR: src/deps-ordered-v3.txt missing — run 'mise run build' first" >&2 exit 2 diff --git a/tasks/test/symbol_order_allowlist.txt b/tasks/test/symbol_order_allowlist.txt new file mode 100644 index 00000000..69a2f16e --- /dev/null +++ b/tasks/test/symbol_order_allowlist.txt @@ -0,0 +1,15 @@ +# One fully-qualified symbol per line (e.g. eql_v3.foo) whose "defined earlier" +# check should be skipped. Use ONLY for genuine false positives (a reference the +# regex sees but that is not a real dependency). Keep this list minimal and +# comment WHY each entry is here. +# +# The two cases that are genuinely not dependencies: +# - a symbol assembled by dynamic SQL (EXECUTE format(...)), which the scan +# reads as a reference but Postgres never resolves at install time; +# - a symbol forward-referenced from a `LANGUAGE plpgsql` body, whose callees +# Postgres resolves at execution time, not at CREATE time. (A `LANGUAGE sql` +# body IS resolved at CREATE time — never allowlist one of those; fix the +# `-- REQUIRE:` edge instead, or the install will fail.) +# +# Adding an entry here weakens the gate that keeps the concatenated installer +# in define-before-use order. Prefer a `-- REQUIRE:` edge whenever one exists. diff --git a/tasks/test/symbol_order_selftest.sh b/tasks/test/symbol_order_selftest.sh new file mode 100755 index 00000000..c1c07dc5 --- /dev/null +++ b/tasks/test/symbol_order_selftest.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +#MISE description="DB-free self-test for the symbol-order cross-check (good passes, mis-ordered fails)" +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT + +# GOOD: definer ordered before user. (a.sql RETURNS a non-owned type — the +# hmac_256 domain-capture branch is exercised separately by d.sql/e.sql below, so +# this pair isolates the eql_v3.eq_term define-before-use ordering it is testing.) +printf 'CREATE FUNCTION eql_v3.eq_term(a public.integer_eq) RETURNS text ...\n' > "$tmp/a.sql" +printf 'CREATE OPERATOR = ( FUNCTION = eql_v3.eq_term );\n' > "$tmp/b.sql" +printf '%s\n%s\n' "$tmp/a.sql" "$tmp/b.sql" > "$tmp/good_order.txt" +bash tasks/test/verify_symbol_order_v3.sh "$tmp/good_order.txt" \ + || { echo "FAIL: good order rejected"; exit 1; } +echo "ok: good order accepted" + +# BAD: user ordered before definer. +printf '%s\n%s\n' "$tmp/b.sql" "$tmp/a.sql" > "$tmp/bad_order.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/bad_order.txt" 2>/dev/null; then + echo "FAIL: mis-ordered reference accepted"; exit 1 +fi +echo "ok: mis-ordered reference rejected" + +# COMMENT-ONLY reference must NOT trip the gate (doxygen @see). +printf -- '--! @see eql_v3.eq_term\nSELECT 1;\n' > "$tmp/c.sql" +printf '%s\n' "$tmp/c.sql" > "$tmp/comment_order.txt" +bash tasks/test/verify_symbol_order_v3.sh "$tmp/comment_order.txt" \ + || { echo "FAIL: comment-only reference tripped the gate"; exit 1; } +echo "ok: comment-only reference ignored" + +# CREATE DOMAIN eql_v3_internal.* form (SEM index-term types hmac_256/ope_cllw/ +# bloom_filter). A domain-form definer ordered before a function returning it +# must be ACCEPTED — pins the domain-capture branch's eql_v3_internal arm so the +# real surface (~165 refs to these three types) can never be misread as "defined +# nowhere" (which would tempt an allowlist entry). +printf 'CREATE DOMAIN eql_v3_internal.hmac_256 AS text;\n' > "$tmp/d.sql" +printf 'CREATE FUNCTION eql_v3.eq_term(a public.integer_eq) RETURNS eql_v3_internal.hmac_256 ...\n' > "$tmp/e.sql" +printf '%s\n%s\n' "$tmp/d.sql" "$tmp/e.sql" > "$tmp/domain_good.txt" +bash tasks/test/verify_symbol_order_v3.sh "$tmp/domain_good.txt" \ + || { echo "FAIL: CREATE DOMAIN eql_v3_internal.* definer not recognised"; exit 1; } +echo "ok: CREATE DOMAIN eql_v3_internal.* definition form recognised" + +# And the same domain-form type used BEFORE it is created must be REJECTED +# (defined-later ordering violation on a SEM index-term type — the exact rot +# this gate exists to catch). +printf '%s\n%s\n' "$tmp/e.sql" "$tmp/d.sql" > "$tmp/domain_bad.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/domain_bad.txt" 2>/dev/null; then + echo "FAIL: eql_v3_internal.hmac_256 used before its CREATE DOMAIN accepted"; exit 1 +fi +echo "ok: domain-form type used before definition rejected" + +# CREATE DOMAIN eql_v3.* form. Query operands live in `eql_v3`, not `public`, +# because a query operand is never a column type: eql_v3.query__ and +# eql_v3.query_json. Pins the domain-capture branch's eql_v3 arm. Without it +# every query domain on the surface reads as "defined nowhere" — the regression +# that reddened every build-dependent CI job. +printf 'CREATE DOMAIN eql_v3.query_integer_eq AS jsonb;\n' > "$tmp/q.sql" +printf 'CREATE FUNCTION eql_v3.eq(a public.integer_eq, b eql_v3.query_integer_eq) ...\n' > "$tmp/qf.sql" +printf '%s\n%s\n' "$tmp/q.sql" "$tmp/qf.sql" > "$tmp/qdomain_good.txt" +bash tasks/test/verify_symbol_order_v3.sh "$tmp/qdomain_good.txt" \ + || { echo "FAIL: CREATE DOMAIN eql_v3.* definer not recognised"; exit 1; } +echo "ok: CREATE DOMAIN eql_v3.* definition form recognised" + +# And a query-operand domain used BEFORE it is created must still be REJECTED — +# proves the eql_v3 arm records a definition rather than silently suppressing the +# symbol (a check that never fires would pass the case above too). +printf '%s\n%s\n' "$tmp/qf.sql" "$tmp/q.sql" > "$tmp/qdomain_bad.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/qdomain_bad.txt" 2>/dev/null; then + echo "FAIL: eql_v3.query_integer_eq used before its CREATE DOMAIN accepted"; exit 1 +fi +echo "ok: query-operand domain used before definition rejected" + +# CREATE OPERATOR CLASS|FAMILY eql_v3_internal.* form (the conditional SEM +# ordered-index opclasses). A file that both creates the opclass and mentions it +# in a RAISE NOTICE (same file) must be ACCEPTED — pins the operator-class +# definition-capture branch so the real ore_block_256/ore_cllw operator_class.sql +# files (self-contained: def + NOTICE prose only) never read as "defined nowhere". +printf "CREATE OPERATOR FAMILY eql_v3_internal.ore_cllw_ops USING btree;\nCREATE OPERATOR CLASS eql_v3_internal.ore_cllw_ops USING btree FAMILY eql_v3_internal.ore_cllw_ops AS STORAGE text;\nRAISE NOTICE 'created operator class eql_v3_internal.ore_cllw_ops';\n" > "$tmp/opclass.sql" +printf '%s\n' "$tmp/opclass.sql" > "$tmp/opclass_order.txt" +bash tasks/test/verify_symbol_order_v3.sh "$tmp/opclass_order.txt" \ + || { echo "FAIL: CREATE OPERATOR CLASS/FAMILY definer not recognised"; exit 1; } +echo "ok: CREATE OPERATOR CLASS/FAMILY definition form recognised" + +# An UNREADABLE path in the ordered list must FAIL the gate, not be silently +# skipped as an empty file (a skipped file's definitions/references go unchecked). +printf '%s\n' "$tmp/does-not-exist.sql" > "$tmp/missing_order.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/missing_order.txt" 2>/dev/null; then + echo "FAIL: unreadable path silently accepted"; exit 1 +fi +echo "ok: unreadable path rejected" + +# An EMPTY ordered list must FAIL, not report "OK (0 files)". A vacuous pass is +# indistinguishable in CI from a real one, so an emptied src/v3 would clear this +# gate and ship an installer containing nothing but the pin script. +: > "$tmp/empty_order.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/empty_order.txt" 2>/dev/null; then + echo "FAIL: empty ordered list passed vacuously"; exit 1 +fi +echo "ok: empty ordered list rejected" + +# Whitespace-only is empty too — the guard must not be fooled by a stray blank line. +printf '\n \n' > "$tmp/blank_order.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/blank_order.txt" 2>/dev/null; then + echo "FAIL: whitespace-only ordered list passed vacuously"; exit 1 +fi +echo "ok: whitespace-only ordered list rejected" + +# An UNREADABLE ALLOWLIST must FAIL the gate. awk's `getline < file` returns <= 0 +# both at EOF and on error, so an unguarded read loop silently yields an empty +# allowlist. That is fail-safe today only because the committed allowlist has no +# active entries; the moment one is added, a path typo would resurrect the very +# false positive the entry exists to suppress — and it would surface inside +# `mise run build`, i.e. inside a release. +if SYMBOL_ORDER_ALLOWLIST="$tmp/no-such-allowlist.txt" \ + bash tasks/test/verify_symbol_order_v3.sh "$tmp/good_order.txt" 2>/dev/null; then + echo "FAIL: unreadable allowlist silently accepted"; exit 1 +fi +echo "ok: unreadable allowlist rejected" + +# The gate is STRICTER than PostgreSQL, deliberately, and the allowlist is the +# escape hatch. A `LANGUAGE plpgsql` body resolves its callees at execution time, +# so Postgres accepts a forward reference that this gate rejects. Pin both halves: +# the rejection (so the strictness is a choice, not an accident) and the release +# valve (so a real forward reference has a documented, reviewable way out). +printf 'CREATE FUNCTION eql_v3.caller() RETURNS int LANGUAGE plpgsql AS $$ BEGIN RETURN eql_v3.callee(); END; $$;\n' > "$tmp/caller.sql" +printf 'CREATE FUNCTION eql_v3.callee() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$;\n' > "$tmp/callee.sql" +printf '%s\n%s\n' "$tmp/caller.sql" "$tmp/callee.sql" > "$tmp/plpgsql_order.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/plpgsql_order.txt" 2>/dev/null; then + echo "FAIL: plpgsql forward reference accepted — the gate's strictness is unpinned"; exit 1 +fi +echo "ok: plpgsql forward reference rejected (documented strictness)" + +printf 'eql_v3.callee # forward-referenced from a plpgsql body\n' > "$tmp/allow.txt" +SYMBOL_ORDER_ALLOWLIST="$tmp/allow.txt" \ + bash tasks/test/verify_symbol_order_v3.sh "$tmp/plpgsql_order.txt" \ + || { echo "FAIL: allowlist did not release the plpgsql forward reference"; exit 1; } +echo "ok: allowlist releases a plpgsql forward reference" + +# --------------------------------------------------------------------------- +# Overload resolution: the gate cannot tell overloads apart (a reference carries +# no argument types), so it must SAY so rather than report a bare OK. On the real +# surface eql_v3.eq has 186 definitions spanning files #55..#242 — keying on +# schema+name and keeping the MIN index means every eq reference after #55 passes +# for free. These three cases pin the boundary of what is still decidable. +# --------------------------------------------------------------------------- + +# (a) Overloaded, and a later overload is still ahead of the reference: NOT +# decidable. Must pass (it is a structural limit, not rot) but must report the +# name as unresolvable instead of claiming a clean check. +printf 'CREATE FUNCTION eql_v3.eq(a public.integer_eq, b public.integer_eq) RETURNS boolean ...\n' > "$tmp/eq_int.sql" +printf 'CREATE OPERATOR = ( FUNCTION = eql_v3.eq, LEFTARG = public.text_eq, RIGHTARG = public.text_eq );\n' > "$tmp/eq_use.sql" +printf 'CREATE FUNCTION eql_v3.eq(a public.text_eq, b public.text_eq) RETURNS boolean ...\n' > "$tmp/eq_text.sql" +printf '%s\n%s\n%s\n' "$tmp/eq_int.sql" "$tmp/eq_use.sql" "$tmp/eq_text.sql" > "$tmp/overload_order.txt" +out="$(bash tasks/test/verify_symbol_order_v3.sh "$tmp/overload_order.txt")" \ + || { echo "FAIL: ambiguous overload treated as an error"; exit 1; } +case "$out" in + *"unresolvable"*) echo "ok: ambiguous overload reported as unresolvable, not a bare OK" ;; + *) echo "FAIL: overload blindness went unreported: [$out]"; exit 1 ;; +esac + +# (b) The reference precedes EVERY definition of the name. Decidable without +# knowing which overload was meant — it is wrong either way. Pins that the +# ambiguity bail-out did not swallow this existing catch. +printf '%s\n%s\n%s\n' "$tmp/eq_use.sql" "$tmp/eq_int.sql" "$tmp/eq_text.sql" > "$tmp/overload_bad.txt" +if bash tasks/test/verify_symbol_order_v3.sh "$tmp/overload_bad.txt" 2>/dev/null; then + echo "FAIL: reference before ALL overloads accepted — the preserved catch is gone"; exit 1 +fi +echo "ok: reference before every overload still rejected" + +# (c) Overloaded but every definition sits in ONE file ordered before the use, so +# the answer is sound whichever overload was meant — must stay fully checked, not +# written off as unresolvable. Mirrors the real eql_v3.ste_vec_contains and the +# CREATE OPERATOR FAMILY + CLASS pair sharing eql_v3_internal.ore_cllw_ops. +printf 'CREATE FUNCTION eql_v3.selector(a public.jsonb_entry) RETURNS text ...\nCREATE FUNCTION eql_v3.selector(a public.jsonb_query) RETURNS text ...\n' > "$tmp/sel_defs.sql" +printf 'SELECT eql_v3.selector(x);\n' > "$tmp/sel_use.sql" +printf '%s\n%s\n' "$tmp/sel_defs.sql" "$tmp/sel_use.sql" > "$tmp/samefile_order.txt" +out="$(bash tasks/test/verify_symbol_order_v3.sh "$tmp/samefile_order.txt")" \ + || { echo "FAIL: same-file overloads rejected"; exit 1; } +case "$out" in + *"unresolvable"*) echo "FAIL: same-file overloads written off as unresolvable: [$out]"; exit 1 ;; + *) echo "ok: overloads all defined before use stay soundly checked" ;; +esac + +echo "symbol-order self-test passed" diff --git a/tasks/test/verify_installer_complete.sh b/tasks/test/verify_installer_complete.sh new file mode 100755 index 00000000..7f0263ef --- /dev/null +++ b/tasks/test/verify_installer_complete.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +#MISE description="Assert release/cipherstash-encrypt.sql contains the body of every ordered src/v3 file (DB-free)" +# +# The order→artefact gate. `eql-codegen order` guarantees the ORDER LIST contains +# every .sql file on disk (pinned by install_order_contains_every_v3_sql_file in +# the eql-codegen parity tests). This gate closes the layer below: that build.sh's +# concat loop actually emitted each ordered file's body into the installer. +# +# Why that layer needs its own gate: 93 of the ~244 files in src/v3 are LEAVES — +# no other file `-- REQUIRE:`s them, and several define no object that any +# inventory test enumerates (a bare `DO` block; functions in eql_v3_internal; a +# `CREATE OPERATOR CLASS`). Drop a leaf and the monolith still applies cleanly and +# every symbol still resolves, so an install smoke test passes. A +# referenced-vs-defined checker (verify_symbol_order_v3.sh) is blind to it by +# construction: a dropped leaf removes its definition AND, being a leaf, leaves no +# reference dangling to trip on. The loss surfaces only in a DB behavioural test — +# which needs CipherStash credentials and is skipped on fork PRs. +# +# So this gate does not look for symbols. It does arithmetic on lines, which no +# leaf can hide from. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +ORDERED="${1:-src/deps-ordered-v3.txt}" +INSTALLER="${2:-release/cipherstash-encrypt.sql}" +PIN="tasks/pin_search_path_v3.sql" + +for f in "$ORDERED" "$INSTALLER" "$PIN"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: $f missing — run 'mise run build' first" >&2 + exit 2 + fi +done + +# `grep -c ''` counts a final line that lacks a trailing newline; `wc -l` does not. +# The trailing-newline gate below makes the two agree, but count the honest way. +# +# grep exits 1 on an empty file (zero lines — legitimate) and >= 2 on a real fault +# (missing, unreadable). Same rc <= 1 idiom as strip_require_lines in +# tasks/build/ordering.sh. A blanket `|| true` would print nothing on a fault, and +# the caller's `$(( expected + total - reqs ))` reads that empty string as 0 — the +# file silently contributes nothing to the identity. Gate 1 makes that unreachable +# today, but this script's whole job is to fail loudly. +# +# Every caller is `var=$(count_lines f)`, so this `exit 2` leaves only the +# command-substitution subshell; the parent aborts because `set -e` sees the +# failed assignment. Keep the callers as bare assignments — `local n=$(...)` or a +# `|| true` would mask the status and restore the silent-zero this replaces. +count_lines() { + local n rc=0 + n=$(grep -c '' "$1") || rc=$? + if (( rc > 1 )); then + echo "ERROR: cannot count lines in $1 (grep exit $rc)" >&2 + exit 2 + fi + echo "${n:-0}" +} + +fail=0 + +# --------------------------------------------------------------------------- +# Gate 1: non-vacuity. The ordered list must name every .sql file on disk. +# +# Without this, an empty or truncated order list sails through every other gate: +# verify_symbol_order_v3.sh prints "OK (0 files)", the self-containment file gate +# finds no offending path, and build.sh emits an installer holding nothing but the +# pin script. Every check green, nothing shipped. Compare against an INDEPENDENT +# find(1) rather than trusting the list's own length. +# --------------------------------------------------------------------------- +echo "==> Non-vacuity gate: the order names every src/v3 SQL file" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +find src/v3 -type f -name '*.sql' ! -name '*_test.sql' | LC_ALL=C sort > "$work/disk" +grep -v '^[[:space:]]*$' "$ORDERED" | LC_ALL=C sort > "$work/order" || true +n_disk=$(count_lines "$work/disk") + +if [[ "$n_disk" -eq 0 ]]; then + echo "ERROR: no .sql files found under src/v3 — refusing to validate an empty surface" >&2 + exit 1 +fi + +# Report at most 10 offenders per direction. An empty order list would otherwise +# print every file in the surface and bury the verdict under 244 lines. +# comm(1) needs real files, not process substitutions: the list is read twice. +report_missing() { + local label=$1 file=$2 n + n=$(count_lines "$file") + if [[ "$n" -gt 0 ]]; then + echo "ERROR: $n file(s) $label:" >&2 + head -10 "$file" | sed 's/^/ /' >&2 + if [[ "$n" -gt 10 ]]; then + echo " … and $(( n - 10 )) more" >&2 + fi + fail=1 + fi + return 0 +} +comm -23 "$work/disk" "$work/order" > "$work/only_disk" +comm -13 "$work/disk" "$work/order" > "$work/only_order" +report_missing "on disk but absent from $ORDERED (they will not ship)" "$work/only_disk" +report_missing "named in $ORDERED but absent from disk" "$work/only_order" +if [[ $fail -eq 0 ]]; then + echo " $n_disk files, order matches disk" +fi + +# --------------------------------------------------------------------------- +# Gate 2: every ordered file ends with a newline. +# +# build.sh assembles with `>>`. A file whose last line has no trailing newline +# would glue its final statement onto the first line of the next file — silently +# producing different SQL, not a syntax error. Nothing else checks this. +# --------------------------------------------------------------------------- +echo "==> Trailing-newline gate: no file can glue onto the next on concat" +while IFS= read -r f; do + [[ -z "$f" ]] && continue + if [[ -s "$f" && -n "$(tail -c1 "$f")" ]]; then + echo "ERROR: $f has no trailing newline — concatenation would merge it with the next file" >&2 + fail=1 + fi +done < "$ORDERED" + +# --------------------------------------------------------------------------- +# Gate 3: line-count identity. +# +# Σ_f (lines(f) − anchored REQUIRE lines(f)) + lines(pin script) == lines(installer) +# +# build.sh strips exactly the anchored `-- REQUIRE:` directives (strip_require_lines +# in tasks/build/ordering.sh) and appends the pin script. So the installer's line +# count is a pure function of the ordered inputs. A dropped file, a truncated body, +# or a duplicated file all break the arithmetic. The REQUIRE regex here MUST match +# strip_require_lines' — keep them in lockstep. +# --------------------------------------------------------------------------- +echo "==> Line-count identity: installer == Σ ordered bodies + pin script" +expected=0 +while IFS= read -r f; do + [[ -z "$f" ]] && continue + total=$(count_lines "$f") + reqs=$(grep -cE '^[[:space:]]*-- REQUIRE:' "$f" || true) + expected=$(( expected + total - reqs )) +done < "$ORDERED" +expected=$(( expected + $(count_lines "$PIN") )) +actual=$(count_lines "$INSTALLER") + +if [[ "$expected" -ne "$actual" ]]; then + echo "ERROR: installer has $actual lines, expected $expected from the ordered inputs" >&2 + echo " (difference of $(( actual - expected )) lines — a file's body was dropped, truncated, or emitted twice)" >&2 + fail=1 +else + echo " $actual lines accounted for" +fi + +if [[ $fail -ne 0 ]]; then + echo "installer completeness gate FAILED" >&2 + exit 1 +fi +echo "installer completeness gate OK" diff --git a/tasks/test/verify_symbol_order_v3.sh b/tasks/test/verify_symbol_order_v3.sh new file mode 100755 index 00000000..1d859d52 --- /dev/null +++ b/tasks/test/verify_symbol_order_v3.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +#MISE description="Cross-check that every eql_v3/eql_v3_internal/public-domain symbol referenced in a file is defined by a file ordered earlier" +# +# This gate is deliberately STRICTER than PostgreSQL, in one direction: it treats +# every owned-schema token as a reference, wherever it appears, including inside a +# function body. PostgreSQL resolves a `LANGUAGE plpgsql` body's callees at +# execution time, so it accepts a plpgsql function that forward-references a +# function defined later in the installer. This gate rejects it. +# +# That is the intended trade: a define-before-use order is what makes a +# single-transaction install of the concatenated monolith safe for `LANGUAGE sql` +# bodies (which Postgres DOES resolve at CREATE time), and the checker cannot tell +# the two languages apart from a line-oriented scan. The cost is that a genuine +# plpgsql forward reference — mutual recursion, say — needs an entry in +# tasks/test/symbol_order_allowlist.txt. +# +# Scope: OVERLOADS ARE NOT RESOLVED HERE. Definitions and references are both +# keyed by schema+name with no argument list, because a reference carries no types +# to key on: a call site is a bare `eql_v3.eq(a, b)`, and CREATE OPERATOR supplies +# LEFTARG/RIGHTARG on other lines. Resolving that needs a type checker, not a +# line-oriented scan. eql_v3.eq has 186 definitions across files #55..#242, so for +# the hot names this gate decides almost nothing — it reports the count of such +# names rather than implying it checked them. +# +# That is not a hole in coverage, because Postgres already resolves overloads +# exactly, at CREATE time, when the concatenated monolith is installed: +# +# mise run test:clean_install_v3 +# +# which runs in CI on every relevant PR across PG 14-17, needs no CipherStash +# credentials, and is not skipped on forks. Verified: swapping text_eq_operators +# ahead of text_eq_functions passes THIS gate and fails that one with +# `function eql_v3.eq(text_eq, text_eq) does not exist`. +# +# So this gate is the DB-free pre-flight; the clean install is the authority. What +# this gate uniquely adds is (1) singleton symbols — hmac_256, the eql_v3.query_* +# domains, the opclasses, version() — where name identifies the object and the +# check is sound, and (2) the plpgsql strictness described above, which the clean +# install cannot catch because Postgres defers those bodies to execution time. +# +# Scope: this checks CROSS-FILE order only. A reference is compared against the +# index of the file that defines it (`defined[tok] > i`), so a symbol referenced +# in the same file that defines it always passes, regardless of line order within +# that file. That is deliberate: the conditional SEM opclass files define an +# operator class and then name it in a RAISE NOTICE in the same file, and several +# generated files reference a domain they just created. Enforcing intra-file order +# would flag all of them and push real definitions onto the allowlist, which is +# the opposite of what the allowlist is for. Postgres resolves within a single +# file's statements in statement order anyway, and that order comes from the +# renderers, not from the install order this gate exists to check. +# +# Note this runs inside `mise run build`, so it gates the release build, not just +# CI. A false positive blocks a release until allowlisted. Both the rejection and +# the allowlist escape hatch are pinned by tasks/test/symbol_order_selftest.sh. +set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +ORDERED="${1:-src/deps-ordered-v3.txt}" +# Overridable so the self-test can exercise the missing-allowlist path without +# disturbing the committed one. +ALLOW="${SYMBOL_ORDER_ALLOWLIST:-tasks/test/symbol_order_allowlist.txt}" +test -f "$ORDERED" || { echo "ERROR: ordered file $ORDERED missing (run mise run build)" >&2; exit 2; } +# Refuse a zero-file run. Without this the checker reports "OK (0 files)" and exits +# 0 on an empty order — a pass that means "I checked nothing", indistinguishable in +# CI from "I checked everything". An emptied surface would sail through here, and +# through the self-containment file gate, into an installer holding only the pin +# script. (A short-but-non-empty order is caught by verify_installer_complete.sh; +# this gate only has to refuse the vacuous case, since the self-test drives it with +# one- and two-file lists.) +grep -qv '^[[:space:]]*$' "$ORDERED" || { echo "ERROR: ordered file $ORDERED is empty — refusing a vacuous check" >&2; exit 2; } +# awk's `getline < file` cannot distinguish EOF from an unreadable file, so an +# unguarded read loop turns a bad ALLOW path into a silently empty allowlist. +# Today that fails safe (nothing to suppress), but this gate runs inside +# `mise run build` — including the release build — so a typo in a future entry +# would resurrect the false positive it was added to suppress, at release time. +test -r "$ALLOW" || { echo "ERROR: allowlist $ALLOW missing or unreadable" >&2; exit 2; } + +awk -v allowfile="$ALLOW" ' + BEGIN { + idx = 0 + while ((getline a < allowfile) > 0) { + sub(/#.*/, "", a); gsub(/[ \t]+/, "", a) + if (a != "") allow[a] = 1 + } + } + # $0 here is a path from the ordered list. + { + idx++ + file = $0 + # Fail loudly on an UNREADABLE path rather than silently treating it as an + # empty (zero-definition) file: getline returns -1 on error but 0 at EOF for + # a genuinely empty file, so only -1 is a fault. This guards both passes — a + # file flagged here sets bad=1, and the END block exits non-zero. + if ((getline probe < file) < 0) { + printf("ERROR: cannot read %s (listed in the ordered file)\n", file) > "/dev/stderr" + bad = 1 + } + close(file) + # First pass over the file: record DEFINITIONS with this index (min index kept). + while ((getline line < file) > 0) { + # Strip trailing line comments so prose/doxygen never counts as code. + sub(/--.*/, "", line) + # CREATE [OR REPLACE] FUNCTION|AGGREGATE eql_v3(_internal). + if (match(line, /CREATE[ \t]+(OR[ \t]+REPLACE[ \t]+)?(FUNCTION|AGGREGATE)[ \t]+(eql_v3_internal|eql_v3)\.("[^"]+"|[a-z0-9_]+)/)) { + s = substr(line, RSTART, RLENGTH); sub(/.*(eql_v3_internal|eql_v3)\./, "", s) + schema = (index(substr(line,RSTART,RLENGTH), "eql_v3_internal.") ? "eql_v3_internal." : "eql_v3.") + key = schema s + record_def(key, idx) + } + # CREATE DOMAIN (eql_v3_internal|eql_v3|public).. All three schemas: the + # SEM index-term types split across DDL forms — hmac_256/ope_cllw/bloom_filter + # are `CREATE DOMAIN eql_v3_internal.` (over text/bytea/smallint[]), + # NOT `CREATE TYPE`. Capturing only `public.` here would leave the three + # most-referenced foundational types (~165 refs) reporting "defined + # nowhere" — a real gap, not an allowlist case. `eql_v3.` owns the + # query-operand domains (`eql_v3.query__`, `eql_v3.query_json`), + # which live outside `public` because a query operand is never a column + # type: omitting the schema here leaves every one of them reporting + # "defined nowhere". Only `public.` domains feed isdomain[] (that gates + # which `public.*` REFERENCES are checked). + # Test eql_v3_internal FIRST in both the alternation and the arms below, so + # the `eql_v3` prefix cannot shadow it. + if (match(line, /CREATE[ \t]+DOMAIN[ \t]+(eql_v3_internal|eql_v3|public)\.[a-z0-9_]+/)) { + seg = substr(line, RSTART, RLENGTH) + if (seg ~ /eql_v3_internal\./) { sub(/.*eql_v3_internal\./, "", seg); key = "eql_v3_internal." seg } + else if (seg ~ /eql_v3\./) { sub(/.*eql_v3\./, "", seg); key = "eql_v3." seg } + else { sub(/.*public\./, "", seg); key = "public." seg; isdomain[seg] = 1 } + record_def(key, idx) + } + # CREATE TYPE eql_v3_internal. (the composite SEM types: ore_block_256, ore_cllw) + if (match(line, /CREATE[ \t]+TYPE[ \t]+eql_v3_internal\.[a-z0-9_]+/)) { + s = substr(line, RSTART, RLENGTH); sub(/.*eql_v3_internal\./, "", s) + key = "eql_v3_internal." s; record_def(key, idx) + } + # CREATE OPERATOR CLASS|FAMILY (eql_v3_internal|eql_v3).. The conditional + # SEM ordered-index opclasses (ore_block_256_operator_class/_family, + # ore_cllw_ops), created via EXECUTE / plpgsql for superusers. Each is fully + # self-contained in its own operator_class.sql — the only other mentions are + # RAISE NOTICE string-literal prose in the SAME file — so recognising this + # definition form (like CREATE TYPE/DOMAIN above) keeps them from reading as + # "defined nowhere", while still catching a genuine cross-file mis-order. + if (match(line, /CREATE[ \t]+OPERATOR[ \t]+(CLASS|FAMILY)[ \t]+(eql_v3_internal|eql_v3)\.[a-z0-9_]+/)) { + s = substr(line, RSTART, RLENGTH) + schema = (index(s, "eql_v3_internal.") ? "eql_v3_internal." : "eql_v3.") + sub(/.*(eql_v3_internal|eql_v3)\./, "", s) + key = schema s; record_def(key, idx) + } + } + close(file) + order[idx] = file + } + END { + # Second pass: for every file, collect REFERENCES (code only) and check them. + for (i = 1; i <= idx; i++) { + file = order[i] + while ((getline line < file) > 0) { + sub(/--.*/, "", line) # drop comments + rest = line + # eql_v3. and eql_v3_internal. + while (match(rest, /(eql_v3_internal|eql_v3)\.("[^"]+"|[a-z0-9_]+)/)) { + tok = substr(rest, RSTART, RLENGTH) + rest = substr(rest, RSTART + RLENGTH) + check(tok, i, file) + } + # public. — ONLY names we saw defined as a domain (avoids public tables/builtins). + rest = line + while (match(rest, /public\.[a-z0-9_]+/)) { + tok = substr(rest, RSTART, RLENGTH); rest = substr(rest, RSTART + RLENGTH) + name = tok; sub(/public\./, "", name) + if (name in isdomain) check(tok, i, file) + } + } + close(file) + } + if (bad) { print "symbol-order cross-check FAILED" > "/dev/stderr"; exit 1 } + # Report the unresolvable set rather than folding it into a bare "OK". A pass + # that says "OK (244 files)" while ~27 overloaded names went unchecked is the + # same lie as the "OK (0 files)" vacuous pass refused above: indistinguishable + # from having actually checked them. + n_unchecked = 0 + for (t in unchecked) n_unchecked++ + if (n_unchecked > 0) { + printf("symbol-order cross-check OK (%d files; %d overloaded name(s) unresolvable here — \ +overload define-before-use is proven exactly by: mise run test:clean_install_v3)\n", idx, n_unchecked) + } else { + print "symbol-order cross-check OK (" idx " files)" + } + } + # Record a definition of `key` at file index `i`. Tracks the min index (the + # ordering check), the max, and the count — the latter two are what let check() + # tell "resolvable" from "overloaded, and I cannot know which one". + function record_def(key, i) { + if (!(key in defined) || i < defined[key]) defined[key] = i + if (!(key in defmax) || i > defmax[key]) defmax[key] = i + defcount[key]++ + } + function check(tok, i, file) { + if (tok in allow) return + if (!(tok in defined)) { + # Referenced owned-schema symbol never defined anywhere: a real hole. + printf("ERROR: %s references %s which is defined nowhere in the installer\n", file, tok) > "/dev/stderr" + bad = 1; return + } + # Reference precedes even the EARLIEST definition of this name. Wrong whichever + # overload was meant, so it is decidable without knowing which one. Checked + # before the ambiguity bail-out below — dropping this would lose a real catch. + if (defined[tok] > i) { + printf("ERROR: %s references %s defined later (at #%d, used at #%d)\n", file, tok, defined[tok], i) > "/dev/stderr" + bad = 1 + return + } + # Overloaded, and at least one overload is still ahead of this reference: the + # right one may or may not be defined yet, and a line-oriented scan cannot say + # which. Bare call sites (`eql_v3.eq(a, b)`) carry no types, and CREATE OPERATOR + # supplies them via LEFTARG/RIGHTARG on other lines. Record, do not guess. + # + # When defmax <= i every overload already precedes the reference, so the answer + # is sound regardless of which one was meant — that keeps the same-file overload + # pairs (eql_v3.ste_vec_contains, eql_v3_internal.compare_ore_block_256_terms) + # fully checked instead of written off. + if (defcount[tok] > 1 && defmax[tok] > i) { + unchecked[tok] = 1 + return + } + } +' "$ORDERED"