From f44097b75df1ab34ec2557b9dddd9e177309aa12 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sat, 18 Jul 2026 08:53:37 -0700 Subject: [PATCH 01/14] engine: broaden LTM site IR to per-occurrence with transform context The reference-site IR (model_ltm_reference_sites) classified access shapes per (from, to) edge, which serves causal-edge emission but not the ceteris-paribus transform: live-ref selection re-derives shapes on printed-and-reparsed Expr0 via the mirror classifier family. The settled Fig. 2 analysis of the round-trips plan established that both consumers ask one per-occurrence question, provided the IR carries context the transform needs and the edge emitter ignores. This extends the walker (one pass, unchanged edge view) with a per-occurrence view over the whole target equation: stable structural SiteIds (deterministic child-index paths), reducer-enclosure context surfaced rather than folded into routing, already-lagged markers (inside PREVIOUS/INIT; suppresses re-wrapping, not selection), index-position markers (the occurrences the transform structurally excludes from live selection), a position-mismatched-iterated per-axis arm distinct from a dynamic index (so the other-dep Collapse/Mismatch/NotIterated verdict is derivable from the IR), and document-order enumeration of module-output composites including implicit SMOOTH/DELAY expansions (the deterministic source A2b needs to fix GH #971's arbitrary HashSet pick). One walker-side correction: a subscript index token that names an element of the source's own dimensions is an element selector, not a causal reference, matching execution (normalize_subscripts3 resolves element-first, position-strict). No consumer-visible edge changes -- variable-level dep extraction already filtered such idents, so the old site was an orphan no keyed consumer read; the fix ends the walker/transform disagreement so A2b can select live occurrences from the IR. Consumers are unchanged in this step; the switch to the occurrence view (and deletion of the Expr0 mirror family) is the follow-on. Track A step 2a of the round-trips plan; part of the #965 arc. --- src/simlin-engine/src/db/ltm_ir.rs | 842 ++++++++++++++++-- src/simlin-engine/src/db/ltm_ir_tests.rs | 704 +++++++++++++++ .../src/ltm_classifier_agreement_tests.rs | 58 +- 3 files changed, 1502 insertions(+), 102 deletions(-) diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index ce71146b9..1e2246412 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -317,15 +317,98 @@ struct WalkCtx<'a> { /// left-to-right DFS over the AST, matching `enumerate_agg_nodes`, so the /// per-source site `Vec`s are deterministic (a salsa requirement on the /// cached IR result). +/// +/// This convenience wrapper discards the [`RawOccurrence`] stream (the finer +/// per-occurrence enumeration) that shares the same single walk; production +/// (`model_ltm_reference_sites`) uses +/// [`collect_all_reference_sites_and_occurrences`] directly. Only the per-edge +/// tests and the A1 classifier-agreement gate consume this narrow view, so it +/// is test-only. +#[cfg(test)] pub(crate) fn collect_all_reference_sites( target_var: &crate::variable::Variable, variables: &HashMap, crate::variable::Variable>, dim_ctx: &crate::dimensions::DimensionsContext, lookup_dims: &mut impl FnMut(&str) -> Vec, ) -> HashMap> { + collect_all_reference_sites_and_occurrences(target_var, variables, dim_ctx, lookup_dims).0 +} + +/// The bundled accumulators the single walk feeds: the per-source +/// [`ReferenceSite`] buckets (the existing per-edge view) and the flat, +/// document-ordered [`RawOccurrence`] stream (the per-occurrence view). The +/// `path` is the running structural [`SiteId`] child-index path (slot prefix +/// plus descent chain); it is `push`/`pop`ed as the walk descends, so at the +/// moment an occurrence is recorded it names exactly that occurrence's node. +struct WalkAccum<'a> { + sites: &'a mut HashMap>, + occurrences: &'a mut Vec, + path: Vec, +} + +impl WalkAccum<'_> { + fn push_ref_site( + &mut self, + from: &str, + shape: RefShape, + target_element: Option<&str>, + reducer_keys: &[String], + ) { + self.sites + .entry(from.to_string()) + .or_default() + .push(ReferenceSite { + shape, + target_element: target_element.map(|s| s.to_string()), + in_reducer: !reducer_keys.is_empty(), + reducer_keys: reducer_keys.to_vec(), + }); + } + + #[allow(clippy::too_many_arguments)] + fn push_occurrence( + &mut self, + reference: OccurrenceRef, + shape: RefShape, + axes: Vec, + target_element: Option<&str>, + reducer_keys: &[String], + already_lagged: bool, + index_nested: bool, + ) { + self.occurrences.push(RawOccurrence { + site_id: SiteId(self.path.clone().into_boxed_slice()), + reference, + shape, + axes, + target_element: target_element.map(|s| s.to_string()), + in_reducer: !reducer_keys.is_empty(), + reducer_keys: reducer_keys.to_vec(), + already_lagged, + index_nested, + }); + } +} + +/// Walk a target's AST once, producing BOTH the per-source [`ReferenceSite`] +/// map (the per-edge view current consumers read) AND the flat, document-order +/// [`RawOccurrence`] stream (the per-occurrence view the ceteris-paribus +/// transform will consume in A2b). Both come from the same single left-to-right +/// DFS, so there is no added pass. The occurrence stream is a superset of the +/// causal references in the map -- it also enumerates module-qualified output +/// composites (which are not model-variable keys) -- and it OMITS an index +/// token that is a literal element selector (the A2a bug fix; see the +/// `Subscript` arm of [`walk_all_in_expr`]). +fn collect_all_reference_sites_and_occurrences( + target_var: &crate::variable::Variable, + variables: &HashMap, crate::variable::Variable>, + dim_ctx: &crate::dimensions::DimensionsContext, + lookup_dims: &mut impl FnMut(&str) -> Vec, +) -> (HashMap>, Vec) { let mut sites: HashMap> = HashMap::new(); + let mut occurrences: Vec = Vec::new(); let Some(ast) = target_var.ast() else { - return sites; + return (sites, occurrences); }; // The target equation's iterated dimensions drive the #511 iterated- // subscript recognition; `Ast::Scalar` has none. @@ -340,85 +423,216 @@ pub(crate) fn collect_all_reference_sites( target_iterated_dims, dim_ctx, }; - match ast { - crate::ast::Ast::Scalar(expr) | crate::ast::Ast::ApplyToAll(_, expr) => { - let mut reducer_keys = Vec::new(); - walk_all_in_expr(expr, &ctx, lookup_dims, None, &mut reducer_keys, &mut sites); - } - crate::ast::Ast::Arrayed(_, subscript_map, default_expr, _) => { - // Per-element expressions: visit slots in canonical element-key - // order so the per-source site Vecs are deterministic. - let mut elem_keys: Vec<_> = subscript_map.keys().collect(); - elem_keys.sort(); - for k in elem_keys { - let mut reducer_keys = Vec::new(); - walk_all_in_expr( - &subscript_map[k], - &ctx, - lookup_dims, - Some(k.as_str()), - &mut reducer_keys, - &mut sites, - ); - } - if let Some(default) = default_expr { + // The `WalkAccum` borrows `sites`/`occurrences`; scope it so those borrows + // end before we move the two out at the end. + { + let mut acc = WalkAccum { + sites: &mut sites, + occurrences: &mut occurrences, + path: Vec::new(), + }; + match ast { + crate::ast::Ast::Scalar(expr) | crate::ast::Ast::ApplyToAll(_, expr) => { + // Slot 0: the single body of a scalar / apply-to-all target. let mut reducer_keys = Vec::new(); + acc.path.push(0); walk_all_in_expr( - default, + expr, &ctx, lookup_dims, None, &mut reducer_keys, - &mut sites, + false, + false, + &mut acc, ); + acc.path.pop(); + } + crate::ast::Ast::Arrayed(_, subscript_map, default_expr, _) => { + // Per-element expressions: visit slots in canonical element-key + // order so the per-source site Vecs (and the occurrence stream / + // its `SiteId`s) are deterministic. The slot index is the first + // `SiteId` path element. + let mut elem_keys: Vec<_> = subscript_map.keys().collect(); + elem_keys.sort(); + for (slot, k) in elem_keys.iter().enumerate() { + let mut reducer_keys = Vec::new(); + acc.path.push(slot as u16); + walk_all_in_expr( + &subscript_map[*k], + &ctx, + lookup_dims, + Some(k.as_str()), + &mut reducer_keys, + false, + false, + &mut acc, + ); + acc.path.pop(); + } + if let Some(default) = default_expr { + // The default expression is the slot after the last element. + let mut reducer_keys = Vec::new(); + acc.path.push(elem_keys.len() as u16); + walk_all_in_expr( + default, + &ctx, + lookup_dims, + None, + &mut reducer_keys, + false, + false, + &mut acc, + ); + acc.path.pop(); + } } } } - sites + (sites, occurrences) +} + +/// If `ident` is a module-qualified output composite (`module·port`, e.g. +/// `mod·out1` or a SMOOTH/DELAY-expanded `$⁚s⁚0⁚smth1·output`) whose head +/// names a model-variable of kind Module, return `(module, port)` (both as +/// they appear in the composite). `module·port` is never itself a +/// model-variable key, so the walker records no `ReferenceSite` for it; this +/// lets the occurrence stream enumerate it as an [`OccurrenceRef::ModuleOutput`] +/// -- the deterministic, document-ordered IR source of truth for the by-name +/// live channel `db::module_link_score_equation` selects today via a +/// per-process-random HashSet `.find()` (GH #971). +fn module_output_parts(ident: &str, ctx: &WalkCtx<'_>) -> Option<(String, String)> { + let pos = ident.find('\u{00B7}')?; + let module = &ident[..pos]; + let port = &ident[pos + '\u{00B7}'.len_utf8()..]; + if port.is_empty() { + return None; + } + let module_ident = Ident::::new(module); + ctx.variables + .get(&module_ident) + .filter(|v| v.is_module()) + .map(|_| (module.to_string(), port.to_string())) } -/// Recursive helper for [`collect_all_reference_sites`]: left-to-right DFS -/// over an `Expr2` tree, pushing one [`ReferenceSite`] per model-variable -/// reference (bucketed by source name). `in_reducer` becomes `true` once we -/// descend into a builtin that can route through an aggregate node and stays -/// sticky (a reducer nested in another reducer's arg is still inside *a* -/// reducer); `SIZE` does not route through an agg, so it never sets the flag. +/// Per-index access classification for a subscript occurrence -- the extended +/// [`OccurrenceAxis`] vocabulary (one entry per index). Each axis reuses the +/// shared [`crate::ltm_agg::classify_axis_access`] classifier (so the reducer +/// and direct-reference paths never disagree) and, where that returns `None`, +/// distinguishes a position-mismatched *iterated* index (a bare `Var` naming a +/// target-iterated dimension that does not line up positionally -- the GH #526 +/// `Mismatch` case) from a genuinely dynamic one (`pop[i+1]`, a range, `@N`). +/// An index that overflows the source's declared arity is handled the same +/// way (an extra iterated-dim name is an arity mismatch, else dynamic). +fn classify_occurrence_axes( + indices: &[crate::ast::IndexExpr2], + source_dims: &[crate::dimensions::Dimension], + target_iterated_dims: &[String], + dim_ctx: &crate::dimensions::DimensionsContext, +) -> Vec { + use crate::ast::{Expr2, IndexExpr2}; + use crate::ltm_agg::classify_axis_access; + + // Mirror `classify_axis_access`'s iterated-dim recognition (a bare `Var` + // whose name is one of the target's iterated dims) so a position-mismatch + // is recorded as `MismatchedIterated`, not collapsed into `Dynamic`. + let mismatched_or_dynamic = |idx: &IndexExpr2| -> OccurrenceAxis { + if let IndexExpr2::Expr(Expr2::Var(name, _, _)) = idx + && target_iterated_dims.iter().any(|t| t == name.as_str()) + { + return OccurrenceAxis::MismatchedIterated { + dim: name.as_str().to_string(), + }; + } + OccurrenceAxis::Dynamic + }; + + indices + .iter() + .enumerate() + .map(|(i, idx)| match source_dims.get(i) { + Some(axis_dim) => { + match classify_axis_access(idx, axis_dim, target_iterated_dims, dim_ctx) { + Some(ar) => OccurrenceAxis::from_axis_read(ar), + None => mismatched_or_dynamic(idx), + } + } + // More indices than declared dims: an arity mismatch. A bare + // iterated-dim name here is `MismatchedIterated`; anything else is + // dynamic (there is no axis to resolve a literal against). + None => mismatched_or_dynamic(idx), + }) + .collect() +} + +/// `true` iff `builtin` is `PREVIOUS(...)` / `INIT(...)`: everything inside is +/// already lagged (read at t-1) or frozen (read at t=0). Used to set the +/// `already_lagged` occurrence marker so the transform does not re-wrap it. +fn builtin_is_previous_or_init(builtin: &crate::builtins::BuiltinFn) -> bool { + matches!( + builtin, + crate::builtins::BuiltinFn::Previous(_, _) | crate::builtins::BuiltinFn::Init(_) + ) +} + +/// Recursive helper for [`collect_all_reference_sites_and_occurrences`]: +/// left-to-right DFS over an `Expr2` tree, pushing one [`ReferenceSite`] per +/// model-variable reference (bucketed by source name) AND one +/// [`RawOccurrence`] per causal reference occurrence. +/// +/// `in_reducer` becomes `true` (via `reducer_keys` non-empty) once we descend +/// into a builtin that can route through an aggregate node and stays sticky (a +/// reducer nested in another reducer's arg is still inside *a* reducer); `SIZE` +/// does not route through an agg, so it never sets the flag. `already_lagged` +/// becomes sticky-true inside a `PREVIOUS`/`INIT` call; `index_nested` +/// becomes sticky-true once we descend into a subscript index expression. +#[allow(clippy::too_many_arguments)] fn walk_all_in_expr( expr: &crate::ast::Expr2, ctx: &WalkCtx<'_>, lookup_dims: &mut impl FnMut(&str) -> Vec, target_element: Option<&str>, reducer_keys: &mut Vec, - sites: &mut HashMap>, + already_lagged: bool, + index_nested: bool, + acc: &mut WalkAccum, ) { use crate::ast::{Expr2, IndexExpr2}; use crate::builtins::{BuiltinContents, walk_builtin_expr}; - let push = |from: &str, - shape: RefShape, - reducer_keys: &[String], - sites: &mut HashMap>| { - sites - .entry(from.to_string()) - .or_default() - .push(ReferenceSite { - shape, - target_element: target_element.map(|s| s.to_string()), - in_reducer: !reducer_keys.is_empty(), - reducer_keys: reducer_keys.to_vec(), - }); - }; - match expr { Expr2::Const(..) => {} Expr2::Var(ident, _, _) => { if ctx.variables.contains_key(ident) { - push(ident.as_str(), RefShape::Bare, reducer_keys, sites); + acc.push_ref_site(ident.as_str(), RefShape::Bare, target_element, reducer_keys); + acc.push_occurrence( + OccurrenceRef::Variable(ident.as_str().to_string()), + RefShape::Bare, + Vec::new(), + target_element, + reducer_keys, + already_lagged, + index_nested, + ); + } else if let Some((module, port)) = module_output_parts(ident.as_str(), ctx) { + acc.push_occurrence( + OccurrenceRef::ModuleOutput { + module, + port, + composite: ident.as_str().to_string(), + }, + RefShape::Bare, + Vec::new(), + target_element, + reducer_keys, + already_lagged, + index_nested, + ); } } Expr2::Subscript(ident, indices, _, _) => { + let from_dims = lookup_dims(ident.as_str()); if ctx.variables.contains_key(ident) { - let from_dims = lookup_dims(ident.as_str()); // #511: an iterated-dimension subscript (`row_sum[Region]` // inside `growth[Region,Age]`) reads the same source element // for the slot being computed -- classify it `Bare` so it @@ -432,21 +646,101 @@ fn walk_all_in_expr( ctx.dim_ctx, ) .unwrap_or_else(|| classify_subscript_shape(indices, &from_dims)); - push(ident.as_str(), shape, reducer_keys, sites); + let axes = classify_occurrence_axes( + indices, + &from_dims, + &ctx.target_iterated_dims, + ctx.dim_ctx, + ); + acc.push_ref_site(ident.as_str(), shape.clone(), target_element, reducer_keys); + acc.push_occurrence( + OccurrenceRef::Variable(ident.as_str().to_string()), + shape, + axes, + target_element, + reducer_keys, + already_lagged, + index_nested, + ); + } else if let Some((module, port)) = module_output_parts(ident.as_str(), ctx) { + acc.push_occurrence( + OccurrenceRef::ModuleOutput { + module, + port, + composite: ident.as_str().to_string(), + }, + RefShape::Bare, + Vec::new(), + target_element, + reducer_keys, + already_lagged, + index_nested, + ); } - for idx in indices { + // Recurse into the indices. An index that resolves to a literal + // element of the subscripted variable's axis is an element + // SELECTOR -- execution resolves it to a static offset, element + // taking priority over any like-named variable + // (`compiler::subscript`, verified by simulation) -- NOT a causal + // reference, so skip it and mint no `elem -> to` site. This ends a + // walker/transform disagreement rather than removing a + // consumer-visible edge: the ceteris-paribus transform already + // treated the token as an element selector, and variable-level dep + // extraction (`variable::classify_dependencies` over the project + // dims) already filtered it, so the pre-fix walker site was an + // orphan no keyed consumer read. It fires only when a subscript + // element name ALSO names a model variable (`arr[nyc]` with a + // variable `nyc`); otherwise nothing was pushed here anyway. + // Everything else is genuine index content: recurse with + // `index_nested = true`, so a model-variable dynamic index + // (`arr[from]`) is marked reachable only through a subscript index. + for (i, idx) in indices.iter().enumerate() { + if resolve_literal_index(idx, &from_dims).is_some() { + continue; + } + acc.path.push(i as u16); match idx { - IndexExpr2::Expr(e) => { - walk_all_in_expr(e, ctx, lookup_dims, target_element, reducer_keys, sites) - } + IndexExpr2::Expr(e) => walk_all_in_expr( + e, + ctx, + lookup_dims, + target_element, + reducer_keys, + already_lagged, + true, + acc, + ), IndexExpr2::Range(l, r, _) => { - walk_all_in_expr(l, ctx, lookup_dims, target_element, reducer_keys, sites); - walk_all_in_expr(r, ctx, lookup_dims, target_element, reducer_keys, sites); + acc.path.push(0); + walk_all_in_expr( + l, + ctx, + lookup_dims, + target_element, + reducer_keys, + already_lagged, + true, + acc, + ); + acc.path.pop(); + acc.path.push(1); + walk_all_in_expr( + r, + ctx, + lookup_dims, + target_element, + reducer_keys, + already_lagged, + true, + acc, + ); + acc.path.pop(); } IndexExpr2::Wildcard(_) | IndexExpr2::StarRange(_, _) | IndexExpr2::DimPosition(_, _) => {} } + acc.path.pop(); } } Expr2::App(builtin, _, _) => { @@ -454,59 +748,119 @@ fn walk_all_in_expr( if pushed_reducer_key { reducer_keys.push(crate::patch::expr2_to_string(expr)); } - walk_builtin_expr(builtin, |contents| match contents { - BuiltinContents::Ident(id, _) => { - if ctx.variables.contains_key(&Ident::::new(id)) { - push(id, RefShape::Bare, reducer_keys, sites); + // Contents of a PREVIOUS/INIT call are already lagged/frozen. + let child_lagged = already_lagged || builtin_is_previous_or_init(builtin); + let mut child: u16 = 0; + walk_builtin_expr(builtin, |contents| { + acc.path.push(child); + match contents { + BuiltinContents::Ident(id, _) => { + let canonical = Ident::::new(id); + if ctx.variables.contains_key(&canonical) { + acc.push_ref_site(id, RefShape::Bare, target_element, reducer_keys); + acc.push_occurrence( + OccurrenceRef::Variable(id.to_string()), + RefShape::Bare, + Vec::new(), + target_element, + reducer_keys, + child_lagged, + index_nested, + ); + } else if let Some((module, port)) = module_output_parts(id, ctx) { + acc.push_occurrence( + OccurrenceRef::ModuleOutput { + module, + port, + composite: id.to_string(), + }, + RefShape::Bare, + Vec::new(), + target_element, + reducer_keys, + child_lagged, + index_nested, + ); + } } + BuiltinContents::Expr(sub_expr) => walk_all_in_expr( + sub_expr, + ctx, + lookup_dims, + target_element, + reducer_keys, + child_lagged, + index_nested, + acc, + ), + // A graphical-function table reference is static data, not + // a causal edge: emit no `from -> consumer` reference site + // for the table itself (only the index argument carries + // real edges). + BuiltinContents::LookupTable(_) => {} } - BuiltinContents::Expr(sub_expr) => walk_all_in_expr( - sub_expr, - ctx, - lookup_dims, - target_element, - reducer_keys, - sites, - ), - // A graphical-function table reference is static data, not a - // causal edge: emit no `from -> consumer` reference site for the - // table itself (only the index argument carries real edges). - BuiltinContents::LookupTable(_) => {} + acc.path.pop(); + child += 1; }); if pushed_reducer_key { reducer_keys.pop(); } } - Expr2::Op1(_, operand, _, _) => walk_all_in_expr( - operand, - ctx, - lookup_dims, - target_element, - reducer_keys, - sites, - ), - Expr2::Op2(_, left, right, _, _) => { - walk_all_in_expr(left, ctx, lookup_dims, target_element, reducer_keys, sites); - walk_all_in_expr(right, ctx, lookup_dims, target_element, reducer_keys, sites); + Expr2::Op1(_, operand, _, _) => { + acc.path.push(0); + walk_all_in_expr( + operand, + ctx, + lookup_dims, + target_element, + reducer_keys, + already_lagged, + index_nested, + acc, + ); + acc.path.pop(); } - Expr2::If(cond, then_e, else_e, _, _) => { - walk_all_in_expr(cond, ctx, lookup_dims, target_element, reducer_keys, sites); + Expr2::Op2(_, left, right, _, _) => { + acc.path.push(0); walk_all_in_expr( - then_e, + left, ctx, lookup_dims, target_element, reducer_keys, - sites, + already_lagged, + index_nested, + acc, ); + acc.path.pop(); + acc.path.push(1); walk_all_in_expr( - else_e, + right, ctx, lookup_dims, target_element, reducer_keys, - sites, + already_lagged, + index_nested, + acc, ); + acc.path.pop(); + } + Expr2::If(cond, then_e, else_e, _, _) => { + for (child, sub) in [cond, then_e, else_e].into_iter().enumerate() { + acc.path.push(child as u16); + walk_all_in_expr( + sub, + ctx, + lookup_dims, + target_element, + reducer_keys, + already_lagged, + index_nested, + acc, + ); + acc.path.pop(); + } } } } @@ -560,6 +914,230 @@ pub(crate) enum SiteRouting { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, salsa::Update)] pub(crate) struct AggRef(pub usize); +// ── Per-occurrence enumeration (Track A2a) ───────────────────────────────── +// +// The per-`(from, to)`-edge [`ClassifiedSite`] view above serves the Expr2 +// edge/routing consumers (`model_element_causal_edges`, `model_edge_shapes`, +// `emit_per_shape_link_scores`). Track A of the "Deleting the Round Trips" +// plan additionally needs the SAME single AST walk to feed the +// ceteris-paribus transform (`ltm_augment.rs`), which today re-derives access +// shape from re-parsed `Expr0`. That consumer operates at a finer granularity +// than a `(from, to)` edge: it selects which *occurrences* of a source stay +// live and PREVIOUS-wraps the rest, per reference occurrence over the WHOLE +// target equation. The [`OccurrenceSite`] records below carry every fact the +// spec's `fig2-answer.md` §3 identifies for that switch (A2b consumes them): +// stable occurrence identity, per-axis access with a mismatched-iterated arm, +// reducer-enclosure / already-lagged / index-position context, and the +// module-qualified by-name live channel. They ride ALONGSIDE `sites`; no +// existing consumer reads them yet. + +/// Stable identity of one reference occurrence within a single target +/// equation: the left-to-right child-index path from the target's slot root +/// down to the occurrence node. +/// +/// The first element is the *slot* index -- for an `Ast::Scalar` / +/// `Ast::ApplyToAll` target the single body is slot `0`; for an +/// `Ast::Arrayed` target the per-element slots are numbered in canonical +/// element-key-sorted order and the (optional) default expression is the +/// slot after the last element. The remaining elements index each child on +/// the descent (operands of `Op1`/`Op2`, branches of `If`, ordered contents +/// of a builtin `App`, and the indices of a `Subscript`). +/// +/// Determinism (a salsa requirement): the walk is a fixed left-to-right DFS +/// with slots visited in sorted key order, so the path is a pure function of +/// the AST -- no HashMap iteration order enters it. Two distinct occurrences +/// can share an access shape but never a `SiteId`, which is what lets the +/// transform name the normalizer (the first non-index-nested matching +/// occurrence) apart from later same-shape ones. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, salsa::Update)] +pub(crate) struct SiteId(pub Box<[u16]>); + +/// What a per-occurrence reference *is* -- the "enumerable as a site at all" +/// decision the spec (§3) requires the unified type to settle once. +/// +/// The walker records an occurrence ONLY for a genuine causal reference. A +/// subscript index that names a dimension element (`arr[nyc]`) is an element +/// *selector* resolved to a static offset at execution +/// (`compiler::subscript`, element takes priority over a like-named +/// variable), NOT a causal reference, so it is enumerated as NO occurrence at +/// all. That keeps the occurrence stream faithful to execution and in step +/// with the ceteris-paribus transform (which already treats the token as a +/// selector); it does not remove a consumer-visible edge, since variable-level +/// dep extraction already filtered the token so no keyed consumer ever read +/// the pre-fix `nyc -> to` walker site. +#[derive(Debug, Clone, PartialEq, Eq, salsa::Update)] +pub(crate) enum OccurrenceRef { + /// A reference to a model variable (the causal `from`). Canonical name. + Variable(String), + /// A module-qualified output composite (`module·port`, including every + /// SMOOTH/DELAY-expanded `$⁚s⁚0⁚smth1·output`). `module·port` is never a + /// model-variable key, so the Expr2 walker records no `ClassifiedSite` for + /// it and `db::module_link_score_equation` locates it BY NAME through a + /// per-process-random `HashSet` `.find()` (GH #971). Enumerating it here, + /// in document order, gives that by-name live channel a deterministic + /// IR source of truth for A2b to consume. + ModuleOutput { + /// Canonical module-instance name (the head before the first `·`). + module: String, + /// Canonical output-port path (everything after the first `·`). + port: String, + /// The full `module·port` composite ident, verbatim. + composite: String, + }, +} + +/// Per-axis access classification for one subscript occurrence, extending +/// [`crate::ltm_agg::AxisRead`] with the position-mismatched-iterated arm the +/// coarse [`RefShape`] loses. +/// +/// A transposed / arity-mismatched iterated index (`arr[D2,D1]` for `arr` +/// declared `[D1,D2]`) yields `None` from `classify_axis_access`, so the +/// subscript collapses to `RefShape::DynamicIndex` -- byte-identical to a +/// genuine dynamic index (`pop[i+1]`). Yet the transform must abandon the +/// changed-first partial for the first (freezing the wrong element is a +/// silent magnitude error, GH #526) while wrapping the second normally. The +/// [`MismatchedIterated`](OccurrenceAxis::MismatchedIterated) arm makes +/// `ltm_augment::classify_other_dep_iterated_dim_subscript`'s +/// `Collapse`/`Mismatch`/`NotIterated` verdict derivable from the IR -- but +/// the per-axis arms alone are NOT the verdict: the derivation also needs the +/// dep's declared arity and the target's iterated-dim count (both of which the +/// consuming transform holds, neither carried on the occurrence). Given a +/// subscript occurrence's `axes`, the target's iterated-dim count `T`, and the +/// dep's declared arity `A` (`None` = un-threadable: the dep is absent from +/// the variable map / has no declared dims), the verdict mirrors +/// `classify_other_dep_iterated_dim_subscript` (`ltm_augment.rs`) exactly: +/// +/// 1. `NotIterated` (normal subscript handling) if `axes` is empty, or +/// `axes.len() > T` (mirrors the `indices.len() > target_iterated_dims.len()` +/// gate, ltm_augment.rs:268), or any axis is `Pinned` / `Reduced` / +/// `Dynamic` (the index is a literal element, wildcard, or dynamic +/// expression, not a bare target-iterated-dim name). +/// 2. otherwise (every axis `Iterated` or `MismatchedIterated`, `axes.len() +/// <= T`): un-threadable dep (`A` is `None`) ⇒ `Collapse` (the transform's +/// permissive fallback, ltm_augment.rs:284); else `axes.len() != A` ⇒ +/// `Mismatch` (the arity check, ltm_augment.rs:288 -- checked BEFORE the +/// per-axis lineup); else all axes `Iterated` ⇒ `Collapse`, ≥1 +/// `MismatchedIterated` ⇒ `Mismatch`. +/// +/// The two arity guards are load-bearing, not decoration: `arr[D1]` for `arr` +/// declared `[D1,D2]` under an A2A-over-`[D1,D2]` target yields all-`Iterated` +/// axes yet is a `Mismatch` (under-arity, corner a); `arr[D1,D1]` for `arr` +/// `[D1,D2]` under an A2A-over-`[D1]` target yields a `MismatchedIterated` axis +/// yet is `NotIterated` (over-target-arity, corner b). Both are pinned by +/// `over_target_arity_iterated_subscript_is_not_iterated` / +/// `under_arity_iterated_subscript_is_mismatch_not_collapse` and the +/// `derive_other_dep_verdict` reference derivation in the tests. One entry per +/// subscript index (empty for a bare `Var` / module output). +#[derive(Debug, Clone, PartialEq, Eq, salsa::Update)] +pub(crate) enum OccurrenceAxis { + /// A literal element of this axis is read (`AxisRead::Pinned`). + Pinned(String), + /// Iterated over the target's dimension space, lined up by name or a + /// positional mapping (`AxisRead::Iterated`). + Iterated { dim: String, source_dim: String }, + /// A reduced axis (`*` / StarRange); present only inside reducer args + /// (`AxisRead::Reduced`). + Reduced { subset: Option> }, + /// Names a target-iterated dimension but does NOT correspond positionally + /// to this source axis (transposed, arity-mismatched, or a mapped pair + /// without a usable positional correspondence). The coarse shape is + /// `DynamicIndex`; the transform must NOT treat it as a genuine dynamic + /// index (GH #526 `Mismatch`). + MismatchedIterated { dim: String }, + /// A genuine dynamic / non-statically-describable index (`pop[i+1]`, a + /// range, `@N`). + Dynamic, +} + +impl OccurrenceAxis { + fn from_axis_read(ar: crate::ltm_agg::AxisRead) -> Self { + use crate::ltm_agg::AxisRead; + match ar { + AxisRead::Pinned(e) => OccurrenceAxis::Pinned(e), + AxisRead::Iterated { dim, source_dim } => OccurrenceAxis::Iterated { dim, source_dim }, + AxisRead::Reduced { subset } => OccurrenceAxis::Reduced { subset }, + } + } +} + +/// How a per-occurrence reference routes, the occurrence-faithful counterpart +/// of [`SiteRouting`]. Unlike `SiteRouting` (which the edge consumers split +/// into one entry per routed agg), a single syntactic occurrence keeps ONE +/// record carrying ALL the synthetic aggs it routes through (GH #793 nested +/// reducers). The RAW walker shape is preserved on the occurrence (the +/// not-hoisted in-reducer `Wildcard`->`DynamicIndex` reclassification the +/// `ClassifiedSite` builder performs is a per-edge-consumer artifact that +/// discards the reducer-enclosure bit; the occurrence keeps `Wildcard` + +/// `in_reducer` instead). +#[derive(Debug, Clone, PartialEq, Eq, salsa::Update)] +pub(crate) enum OccurrenceRouting { + /// Consumers use the occurrence's own `shape` / `axes`. + Direct, + /// The occurrence's read is carried by one or more synthetic aggregate + /// nodes (its enclosing hoisted reducer(s)). + ThroughAgg { aggs: Vec }, +} + +/// One classified reference occurrence over a target equation, the finer +/// substrate both LTM consumers project from (edge emission is a per-edge +/// dedup of these; the transform selects a live SET by shape and names the +/// first non-index-nested occurrence as the normalizer). Every field maps to +/// a spec §3 requirement with a named A2b/A3 consumer -- there are no +/// speculative fields. +#[derive(Debug, Clone, PartialEq, Eq, salsa::Update)] +pub(crate) struct OccurrenceSite { + /// Stable, deterministic occurrence identity within `to`'s equation. + pub site_id: SiteId, + /// The reference (a model variable, or a module-qualified output). + pub reference: OccurrenceRef, + /// The coarse per-reference access shape -- classified EXACTLY as the + /// `ClassifiedSite` path does (the raw walker shape, before the per-edge + /// `Wildcard`->`DynamicIndex` reclassification), so the two views agree. + pub shape: RefShape, + /// Per-index access (the extended `AxisRead` vocabulary). Empty for a bare + /// `Var` / module output. + pub axes: Vec, + /// `Some(elem)` when the occurrence sits in an `Ast::Arrayed` per-element + /// slot (canonical element / comma-tuple), else `None`. + pub target_element: Option, + /// How the occurrence routes (direct, or through its hoisted reducer aggs). + pub routing: OccurrenceRouting, + /// `true` iff the occurrence sits syntactically inside an aggregate-routed + /// reducer (`SUM`/`MEAN`/`MIN`/`MAX`/`STDDEV`/`RANK`). Surfaced explicitly + /// -- the transform's whole-reducer freeze (#517) and bare-feeder decline + /// (#779) need "inside a scalar reducer" even when routing is `Direct`. + pub in_reducer: bool, + /// Canonical printed text of every enclosing hoistable reducer, outermost + /// to innermost (empty when `!in_reducer`). + pub reducer_keys: Vec, + /// `true` iff the occurrence sits inside a `PREVIOUS(...)` / `INIT(...)` + /// call -- already lagged/frozen. The transform must NOT re-wrap it (a + /// double lag reads t-2), though it stays live-selectable (Q3). The edge + /// emitter ignores this. + pub already_lagged: bool, + /// `true` iff the occurrence is reachable ONLY through another reference's + /// subscript index (`other_arr[from]`). Such an occurrence is excluded + /// from live selection, from the normalizer, and from the changed-last + /// freeze, and turns a reducer whose only live occurrence is index-nested + /// into a whole-reducer freeze (Q4). The edge emitter ignores this. + pub index_nested: bool, +} + +/// Walker output for one occurrence, before the (agg-dependent) routing is +/// resolved -- the occurrence analogue of [`ReferenceSite`]. `model_ltm_reference_sites` +/// finalizes it into an [`OccurrenceSite`] by attaching [`OccurrenceRouting`]. +struct RawOccurrence { + site_id: SiteId, + reference: OccurrenceRef, + shape: RefShape, + axes: Vec, + target_element: Option, + in_reducer: bool, + reducer_keys: Vec, + already_lagged: bool, + index_nested: bool, +} + /// The reference-site classification for a model: every `(from-var, to-var)` /// causal edge with ≥1 AST reference, mapped to its classified sites. /// @@ -574,9 +1152,20 @@ pub(crate) struct AggRef(pub usize); /// reference) simply has *no* entry here -- consumers fall back to a single /// `Bare` site for it, exactly as the pre-IR walkers' `is_empty()` / /// module pre-checks did. +/// +/// `occurrences` is the finer, per-reference-occurrence enumeration over the +/// whole target equation (Track A2a), keyed by TARGET canonical name; each +/// value `Vec` is in stable left-to-right DFS order (slots in +/// sorted key order). It is a *superset* of `sites`' causal references -- it +/// also enumerates the module-qualified by-name live channel (`OccurrenceRef::ModuleOutput`) +/// that has no `ClassifiedSite`. It rides alongside `sites`; no current +/// consumer reads it (A2b is the first). Like `sites`, the HashMap *key* +/// order is irrelevant (consumers sort keys themselves); only each value +/// `Vec`'s order is load-bearing for salsa determinism. #[derive(Debug, Clone, Default, PartialEq, Eq, salsa::Update)] pub(crate) struct LtmReferenceSitesResult { pub sites: HashMap<(String, String), Vec>, + pub occurrences: HashMap>, } /// Classify every causal-edge reference site in `model` exactly once. @@ -639,14 +1228,25 @@ pub(crate) fn model_ltm_reference_sites( to_names.sort(); let mut sites: HashMap<(String, String), Vec> = HashMap::new(); + let mut occurrences: HashMap> = HashMap::new(); for to_name in to_names { let to_var = &variables[to_name]; let to_name_str = to_name.as_str(); - let raw_by_source = - collect_all_reference_sites(to_var, &variables, dim_ctx, &mut lookup_dims); - if raw_by_source.is_empty() { + // One walk feeds BOTH views: the per-source `ReferenceSite` buckets + // (per-edge, existing consumers) and the flat, document-ordered + // `RawOccurrence` stream (per-occurrence, the A2b transform). + let (raw_by_source, raw_occurrences) = collect_all_reference_sites_and_occurrences( + to_var, + &variables, + dim_ctx, + &mut lookup_dims, + ); + // A target that references ONLY module-qualified outputs has no + // `ReferenceSite` (module·port is not a model-variable key) but does + // carry occurrences, so gate the skip on both being empty. + if raw_by_source.is_empty() && raw_occurrences.is_empty() { continue; } @@ -663,6 +1263,60 @@ pub(crate) fn model_ltm_reference_sites( }) .unwrap_or_default(); + // The synthetic aggs a `from` reference routes through, narrowed to + // the aggs minted for one of the reference's *enclosing* reducers -- + // the exact per-occurrence `ThroughAgg` decision the `ClassifiedSite` + // loop below makes per raw site (GH #793), reused for the occurrence + // stream's `OccurrenceRouting`. + let matching_aggs_for = |from_name: &str, reducer_keys: &[String]| -> Vec { + synthetic_aggs_in_to + .iter() + .copied() + .filter(|&i| agg_nodes.aggs[i].reads_var(from_name)) + .filter(|&i| { + reducer_keys + .iter() + .any(|k| k == &agg_nodes.aggs[i].equation_text) + }) + .map(AggRef) + .collect() + }; + + // Finalize the per-occurrence view: attach routing (a single occurrence + // keeps ONE record carrying all its matching aggs, unlike the per-edge + // `ClassifiedSite` which duplicates one entry per agg). The RAW walker + // shape is preserved -- the not-hoisted in-reducer `Wildcard`->`DynamicIndex` + // reclassification below is a per-edge-consumer artifact. + let mut occ_sites: Vec = Vec::with_capacity(raw_occurrences.len()); + for occ in raw_occurrences { + let routing = match &occ.reference { + OccurrenceRef::Variable(from) if occ.in_reducer => { + let aggs = matching_aggs_for(from, &occ.reducer_keys); + if aggs.is_empty() { + OccurrenceRouting::Direct + } else { + OccurrenceRouting::ThroughAgg { aggs } + } + } + _ => OccurrenceRouting::Direct, + }; + occ_sites.push(OccurrenceSite { + site_id: occ.site_id, + reference: occ.reference, + shape: occ.shape, + axes: occ.axes, + target_element: occ.target_element, + routing, + in_reducer: occ.in_reducer, + reducer_keys: occ.reducer_keys, + already_lagged: occ.already_lagged, + index_nested: occ.index_nested, + }); + } + if !occ_sites.is_empty() { + occurrences.insert(to_name_str.to_string(), occ_sites); + } + for (from_name, raw_sites) in raw_by_source { // Synthetic aggs of `to` that read `from`. The per-site routing // below further narrows this by canonical reducer text; a sibling @@ -765,7 +1419,7 @@ pub(crate) fn model_ltm_reference_sites( } } - LtmReferenceSitesResult { sites } + LtmReferenceSitesResult { sites, occurrences } } #[cfg(test)] diff --git a/src/simlin-engine/src/db/ltm_ir_tests.rs b/src/simlin-engine/src/db/ltm_ir_tests.rs index c4b450bd4..b03632697 100644 --- a/src/simlin-engine/src/db/ltm_ir_tests.rs +++ b/src/simlin-engine/src/db/ltm_ir_tests.rs @@ -890,3 +890,707 @@ mod model_ltm_reference_sites_tests { }); } } + +// ── Layer 3: the per-occurrence enumeration (Track A2a) ──────────────────── +// +// The `occurrences` view is the finer, per-reference-occurrence enumeration +// over a whole target equation that the ceteris-paribus transform will consume +// in A2b. Each test drives a hard shape and pins the one field it adds; every +// pin would fail without the corresponding addition. +mod occurrence_ir_tests { + use super::*; + use crate::datamodel; + use crate::db::ltm_ir::{OccurrenceAxis, OccurrenceRef, OccurrenceRouting, OccurrenceSite}; + use crate::testutils::{x_aux, x_flow, x_model, x_module, x_stock}; + + /// Sync `datamodel`, run `model_ltm_reference_sites`, and return the + /// occurrence stream for `target` in `model_name` (empty if none). + fn occ_from_datamodel( + datamodel: &datamodel::Project, + model_name: &str, + target: &str, + ) -> Vec { + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, datamodel); + let model = sync.models[model_name].source; + let proj = sync.project; + let ir = model_ltm_reference_sites(&db, model, proj); + ir.occurrences.get(target).cloned().unwrap_or_default() + } + + /// `occ_from_datamodel` for the `main` model of a `TestProject`. + fn occ_of(project: &TestProject, target: &str) -> Vec { + occ_from_datamodel(&project.build_datamodel(), "main", target) + } + + /// Run `model_ltm_reference_sites` for `project`'s `main` model and hand the + /// full IR plus `target`'s occurrence stream to `body` (for tests that need + /// to cross-check the occurrence view against the per-edge `sites` view). + fn with_ir_and_occ( + project: &TestProject, + target: &str, + body: impl FnOnce(&LtmReferenceSitesResult, &[OccurrenceSite]), + ) { + let datamodel = project.build_datamodel(); + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, &datamodel); + let model = sync.models["main"].source; + let proj = sync.project; + let ir = model_ltm_reference_sites(&db, model, proj); + let occs = ir.occurrences.get(target).cloned().unwrap_or_default(); + body(ir, &occs); + } + + fn var_occs<'a>(occs: &'a [OccurrenceSite], name: &str) -> Vec<&'a OccurrenceSite> { + occs.iter() + .filter(|o| matches!(&o.reference, OccurrenceRef::Variable(v) if v == name)) + .collect() + } + + #[test] + fn enumerates_every_occurrence_with_stable_distinct_site_ids() { + // `relative_pop[Region] = population / population[nyc]`: two occurrences + // of `population` -- the bare numerator (document-first) then the + // FixedIndex denominator -- enumerated per occurrence over the whole + // equation, with distinct, deterministic `SiteId`s. + let project = TestProject::new("main") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("population[Region]", "100") + .array_aux("relative_pop[Region]", "population / population[nyc]"); + let occs = occ_of(&project, "relative_pop"); + let pop = var_occs(&occs, "population"); + assert_eq!(pop.len(), 2, "occs: {occs:?}"); + assert_eq!(pop[0].shape, RefShape::Bare, "numerator is document-first"); + assert_eq!(pop[1].shape, RefShape::FixedIndex(vec!["nyc".to_string()])); + assert_ne!( + pop[0].site_id, pop[1].site_id, + "two same-source occurrences must have distinct SiteIds" + ); + // Determinism (a salsa requirement): a fresh build yields the identical + // stream, SiteIds included. + let occs2 = occ_of(&project, "relative_pop"); + assert_eq!( + occs, occs2, + "the occurrence stream (including SiteIds) must be deterministic" + ); + } + + #[test] + fn reducer_enclosure_surfaced_even_when_routing_is_direct() { + // `z = SUM(pop[idx, *])` with `idx` a scalar: the reducer is NOT hoisted + // (dynamic index), so the occurrence routes `Direct` -- yet it must + // still surface `in_reducer` + `reducer_keys` (the #517/#779 + // "inside a scalar reducer" fact the per-edge `ClassifiedSite` folds + // away by reclassifying Wildcard->DynamicIndex and dropping the bit). + let project = TestProject::new("main") + .named_dimension("D1", &["a", "b"]) + .named_dimension("D2", &["x", "y"]) + .array_aux("pop[D1, D2]", "1") + .scalar_aux("idx", "1") + .scalar_aux("z", "SUM(pop[idx, *])"); + with_ir_and_occ(&project, "z", |ir, occs| { + let pop = var_occs(occs, "pop"); + assert_eq!(pop.len(), 1, "occs: {occs:?}"); + let o = pop[0]; + assert_eq!( + o.shape, + RefShape::Wildcard, + "the occurrence keeps the RAW walker shape, unlike ClassifiedSite" + ); + assert!(o.in_reducer, "the SUM argument sits inside a reducer"); + assert_eq!(o.reducer_keys.len(), 1, "one enclosing reducer"); + assert_eq!( + o.routing, + OccurrenceRouting::Direct, + "the dynamic-index reducer is not hoisted" + ); + // The per-edge view folds the enclosure away: it reclassifies to + // DynamicIndex, discarding the reducer-enclosure bit. + let sites = ir.sites.get(&("pop".to_string(), "z".to_string())).unwrap(); + assert!( + sites.iter().any(|s| s.shape == RefShape::DynamicIndex), + "ClassifiedSite reclassifies the unhoisted Wildcard to DynamicIndex" + ); + }); + } + + #[test] + fn hoisted_reducer_occurrence_routes_through_agg() { + // `share[Region] = pop / SUM(pop[*])`: the bare numerator is Direct/Bare; + // the SUM's wildcard arg is hoisted, so its occurrence routes ThroughAgg + // while KEEPING the raw Wildcard shape. + let project = TestProject::new("main") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "100") + .array_aux("share[Region]", "pop / SUM(pop[*])"); + with_ir_and_occ(&project, "share", |_ir, occs| { + let pop = var_occs(occs, "pop"); + assert_eq!(pop.len(), 2, "occs: {occs:?}"); + let bare = pop.iter().find(|o| !o.in_reducer).expect("bare numerator"); + assert_eq!(bare.routing, OccurrenceRouting::Direct); + assert_eq!(bare.shape, RefShape::Bare); + let reduced = pop.iter().find(|o| o.in_reducer).expect("SUM argument"); + assert!( + matches!(reduced.routing, OccurrenceRouting::ThroughAgg { .. }), + "a hoisted reducer occurrence routes through its synthetic agg" + ); + assert_eq!( + reduced.shape, + RefShape::Wildcard, + "the raw walker shape is preserved on the occurrence" + ); + }); + } + + #[test] + fn already_lagged_marks_previous_and_init_contents() { + // `to = from + PREVIOUS(g) + INIT(h)`: `from` is live/unlagged; the + // occurrences inside PREVIOUS and INIT are `already_lagged` (so the + // transform will not re-wrap and double-lag them). + let project = TestProject::new("main") + .scalar_aux("from", "1") + .scalar_aux("g", "2") + .scalar_aux("h", "3") + .scalar_aux("to", "from + PREVIOUS(g) + INIT(h)"); + with_ir_and_occ(&project, "to", |_ir, occs| { + let get = |name: &str| { + var_occs(occs, name) + .first() + .copied() + .unwrap_or_else(|| panic!("no occurrence for {name}: {occs:?}")) + }; + assert!(!get("from").already_lagged, "the bare `from` is not lagged"); + assert!(get("g").already_lagged, "g sits inside PREVIOUS"); + assert!(get("h").already_lagged, "h sits inside INIT"); + }); + } + + #[test] + fn index_nested_marks_subscript_index_occurrence() { + // `to = SUM(w[from]) + from`: the `from` inside `w[from]` is reachable + // ONLY through a subscript index (`index_nested`), while the bare `from` + // outside is not -- the distinction the transform needs to name the + // normalizer and to whole-freeze a reducer whose only live occurrence is + // index-nested (Q4). + let project = TestProject::new("main") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("w[Region]", "1") + .scalar_aux("from", "1") + .scalar_aux("to", "SUM(w[from]) + from"); + with_ir_and_occ(&project, "to", |_ir, occs| { + let froms = var_occs(occs, "from"); + assert_eq!(froms.len(), 2, "occs: {occs:?}"); + let nested = froms + .iter() + .find(|o| o.index_nested) + .expect("the from inside w[from]"); + assert!(nested.in_reducer, "w[from] sits inside SUM"); + let bare = froms + .iter() + .find(|o| !o.index_nested) + .expect("the bare `+ from`"); + assert!(!bare.in_reducer, "the bare `from` is outside the reducer"); + }); + } + + #[test] + fn mismatched_iterated_axis_is_distinct_from_dynamic() { + // A transposed subscript `arr[D2, D1]` (arr declared `[D1, D2]`) inside + // an A2A-over-`[D1, D2]` target: the coarse shape collapses to + // DynamicIndex (unchanged), but the per-axis record marks each axis + // `MismatchedIterated`, so `classify_other_dep_iterated_dim_subscript`'s + // `Mismatch` verdict is derivable from the IR (distinct from a genuine + // dynamic index below). + let project = TestProject::new("main") + .named_dimension("D1", &["a", "b"]) + .named_dimension("D2", &["x", "y"]) + .array_aux("arr[D1, D2]", "1") + .array_aux_direct( + "t", + vec!["D1".to_string(), "D2".to_string()], + "arr[D2, D1] * 2", + None, + ); + with_ir_and_occ(&project, "t", |_ir, occs| { + let arr = var_occs(occs, "arr"); + assert_eq!(arr.len(), 1, "occs: {occs:?}"); + assert_eq!( + arr[0].shape, + RefShape::DynamicIndex, + "coarse shape unchanged" + ); + assert_eq!( + arr[0].axes, + vec![ + OccurrenceAxis::MismatchedIterated { + dim: "d2".to_string() + }, + OccurrenceAxis::MismatchedIterated { + dim: "d1".to_string() + }, + ], + "a transposed iterated subscript is per-axis MismatchedIterated" + ); + }); + + // A genuinely dynamic index (`pp[k + 1]`) is `Dynamic`, not + // MismatchedIterated -- the distinction the transform must act on. + let dyn_project = TestProject::new("main") + .indexed_dimension("Idx", 3) + .array_aux("pp[Idx]", "1") + .scalar_aux("k", "1") + .array_aux_direct("dref", vec!["Idx".to_string()], "pp[k + 1]", None); + with_ir_and_occ(&dyn_project, "dref", |_ir, occs| { + let pp = var_occs(occs, "pp"); + assert_eq!(pp.len(), 1, "occs: {occs:?}"); + assert_eq!(pp[0].shape, RefShape::DynamicIndex); + assert_eq!( + pp[0].axes, + vec![OccurrenceAxis::Dynamic], + "an arithmetic index is Dynamic, never MismatchedIterated" + ); + }); + } + + /// A multi-output module whose parent aux reads TWO of its output ports: + /// the module-qualified live channel `db::module_link_score_equation` + /// selects today via a per-process-random HashSet `.find()` (GH #971). The + /// occurrence stream enumerates both composites in document order so A2b has + /// a deterministic IR source of truth. + fn two_output_module_project() -> datamodel::Project { + datamodel::Project { + name: "two_output".to_string(), + sim_specs: datamodel::SimSpecs { + start: 0.0, + stop: 1.0, + dt: datamodel::Dt::Dt(1.0), + save_step: None, + sim_method: datamodel::SimMethod::Euler, + time_units: None, + }, + dimensions: vec![], + units: vec![], + models: vec![ + x_model( + "main", + vec![ + x_stock("level", "50", &[], &["adjustment"], None), + // `combined` reads out_a THEN out_b (document order). + x_aux("combined", "multi_out.out_a + multi_out.out_b", None), + x_flow("adjustment", "combined / 5", None), + x_module("multi_out", &[("level", "multi_out.input")], None), + ], + ), + x_model( + "multi_out", + vec![ + datamodel::Variable::Aux(datamodel::Aux { + ident: "input".to_string(), + equation: datamodel::Equation::Scalar("0".to_string()), + documentation: String::new(), + units: None, + gf: None, + ai_state: None, + uid: None, + compat: datamodel::Compat { + can_be_module_input: true, + ..datamodel::Compat::default() + }, + }), + x_aux("out_a", "input * 2", None), + x_aux("out_b", "input * 3", None), + ], + ), + ], + source: None, + ai_information: None, + } + } + + #[test] + fn module_output_occurrences_recorded_in_document_order() { + let project = two_output_module_project(); + let occs = occ_from_datamodel(&project, "main", "combined"); + let module_occs: Vec<(String, String, String)> = occs + .iter() + .filter_map(|o| match &o.reference { + OccurrenceRef::ModuleOutput { + module, + port, + composite, + } => Some((module.clone(), port.clone(), composite.clone())), + _ => None, + }) + .collect(); + assert_eq!( + module_occs, + vec![ + ( + "multi_out".to_string(), + "out_a".to_string(), + "multi_out\u{00B7}out_a".to_string() + ), + ( + "multi_out".to_string(), + "out_b".to_string(), + "multi_out\u{00B7}out_b".to_string() + ), + ], + "both module-output composites are enumerated in document order \ + (out_a before out_b); no ClassifiedSite exists for these" + ); + } + + /// The dominant production shape: an IMPLICIT stdlib expansion. `smoothed = + /// SMTH1(input, 5) * 2` desugars (in the builtins visitor) to an implicit + /// `Variable::Module` named `$⁚smoothed⁚0⁚smth1` whose `·output` composite + /// the rewritten parent equation reads. Where + /// `module_output_occurrences_recorded_in_document_order` pins an EXPLICIT + /// author-written multi-output module, this pins the implicit path -- and it + /// needs its own pin because it ADDITIONALLY depends on + /// `reconstruct_model_variables`' implicit-var loop reconstructing that + /// SMOOTH-expanded `Variable::Module`. `module_output_parts` only enumerates + /// a `·`-composite whose head resolves to a module-kind variable in the + /// reconstructed map; if a future change to that loop stopped rebuilding the + /// implicit modules, the composite head would go unresolved and this channel + /// would empty silently for the case that dominates real models -- with the + /// explicit-module pin still green. Direct routing: the composite read sits + /// outside any reducer. + #[test] + fn implicit_stdlib_module_output_occurrence_recorded_in_document_order() { + let project = TestProject::new("main") + .scalar_aux("input", "3") + .scalar_aux("smoothed", "SMTH1(input, 5) * 2"); + let occs = occ_of(&project, "smoothed"); + let module_occs: Vec<(String, String, String)> = occs + .iter() + .filter_map(|o| match &o.reference { + OccurrenceRef::ModuleOutput { + module, + port, + composite, + } => Some((module.clone(), port.clone(), composite.clone())), + _ => None, + }) + .collect(); + assert_eq!( + module_occs, + vec![( + "$\u{205A}smoothed\u{205A}0\u{205A}smth1".to_string(), + "output".to_string(), + "$\u{205A}smoothed\u{205A}0\u{205A}smth1\u{00B7}output".to_string(), + )], + "the SMTH1 expansion's `·output` composite is enumerated exactly once \ + as a ModuleOutput occurrence; no ClassifiedSite exists for it: {occs:?}" + ); + let module_occ = occs + .iter() + .find(|o| matches!(&o.reference, OccurrenceRef::ModuleOutput { .. })) + .expect("the module-output occurrence"); + assert_eq!( + module_occ.routing, + OccurrenceRouting::Direct, + "the composite read is not inside a reducer" + ); + } + + #[test] + fn element_selector_index_is_not_a_causal_occurrence() { + // Runtime pin: `pick = population[nyc]` reads the ELEMENT even though a + // variable `nyc = 2` exists. Element interpretation -> population's nyc + // slot (100); a variable-index interpretation would read population[2] + // (boston, 200). Simulation confirms 100. + let project = TestProject::new("main") + .with_sim_time(0.0, 1.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .array_with_ranges_direct( + "population", + vec!["Region".to_string()], + vec![("nyc", "100"), ("boston", "200")], + None, + ) + .scalar_aux("nyc", "2") + .scalar_aux("pick", "population[nyc]"); + assert_eq!( + project.vm_result_incremental("pick")[0], + 100.0, + "execution reads the element population[nyc]=100, not variable nyc" + ); + // Edge-set pin: the walker and the ceteris-paribus transform now AGREE + // the element selector is not a site, keeping the occurrence stream A2b + // live-selects from faithful to execution. This is NOT the removal of a + // consumer-visible causal edge: variable-level dep extraction + // (`variable.rs` `ClassifyVisitor::is_dimension_or_element`, over all + // project dims) already filtered the element-colliding index ident, so + // the pre-fix walker's `nyc -> pick` site was an orphan no keyed + // consumer ever read -- no `nyc -> pick` edge existed before either. + // Only `population -> pick` FixedIndex remains. + with_ir_and_occ(&project, "pick", |ir, occs| { + assert!( + var_occs(occs, "nyc").is_empty(), + "the element selector `nyc` in population[nyc] is not a causal \ + occurrence: {occs:?}" + ); + assert!( + !ir.sites + .contains_key(&("nyc".to_string(), "pick".to_string())), + "no spurious nyc -> pick causal edge" + ); + assert!( + ir.sites + .contains_key(&("population".to_string(), "pick".to_string())), + "the real population -> pick edge remains" + ); + }); + } + + #[test] + fn colliding_element_index_skipped_but_bare_variable_kept() { + // `collide[Region] = population[nyc] * nyc`: the index `nyc` is an + // element selector (skipped), but the bare `* nyc` is a genuine variable + // reference (kept). Exactly one `nyc` occurrence, and it is not + // index-nested. + let project = TestProject::new("main") + .named_dimension("Region", &["nyc", "boston"]) + .scalar_aux("nyc", "3") + .array_aux("population[Region]", "100") + .array_aux("collide[Region]", "population[nyc] * nyc"); + with_ir_and_occ(&project, "collide", |ir, occs| { + let nyc = var_occs(occs, "nyc"); + assert_eq!( + nyc.len(), + 1, + "only the bare `* nyc` is causal, not the element selector: {occs:?}" + ); + assert!( + !nyc[0].index_nested, + "the surviving nyc occurrence is the bare multiplication" + ); + let sites = ir + .sites + .get(&("nyc".to_string(), "collide".to_string())) + .expect("the bare nyc -> collide edge"); + assert_eq!(sites.len(), 1, "one Bare nyc site, not the pre-fix two"); + assert_eq!(sites[0].shape, RefShape::Bare); + }); + } + + // ── Other-dep verdict derivation (the A2b contract on `axes`) ─────────── + // + // `OccurrenceAxis`'s rustdoc states the rule by which A2b will derive + // `ltm_augment::classify_other_dep_iterated_dim_subscript`'s + // `Collapse`/`Mismatch`/`NotIterated` verdict from an occurrence's `axes` + // (after deleting the Expr0 mirror classifier). The tests below are the + // executable form of that contract: [`derive_other_dep_verdict`] is the + // reference derivation, and the two corner tests pin the `axes` the walker + // actually produces for the arity shapes where a rule keyed on the per-axis + // arms ALONE (all-`Iterated` ⇒ Collapse, any-`MismatchedIterated` ⇒ + // Mismatch) diverges from the transform. Deriving the verdict requires the + // two arity facts the occurrence does not itself carry -- the dep's declared + // arity and the target's iterated-dim count -- both of which A2b holds. + + /// The three verdicts of `ltm_augment::OtherDepIteratedVerdict`, mirrored + /// here so the derivation contract is testable without reaching into + /// `ltm_augment`'s private types. + #[derive(Debug, PartialEq, Eq)] + enum DerivedVerdict { + Collapse, + Mismatch, + NotIterated, + } + + /// Reference implementation of the `OccurrenceAxis` rustdoc's derivation + /// rule -- the verdict A2b must compute for a subscript occurrence's + /// `axes`, given the dep's declared arity (`None` = un-threadable: the dep + /// is absent from the variable map / has no declared dims) and the target's + /// iterated-dim count. It reproduces + /// `classify_other_dep_iterated_dim_subscript` (`ltm_augment.rs`) exactly, + /// including the two arity guards a per-axis-only rule would miss: + /// - `axes.len() > target_iterated_count` ⇒ `NotIterated` (mirrors the + /// `indices.len() > target_iterated_dims.len()` gate, ltm_augment.rs:268); + /// - a threadable dep whose declared arity differs from `axes.len()` ⇒ + /// `Mismatch` (mirrors `index_dims.len() != dep_dims.len()`, + /// ltm_augment.rs:288) -- checked BEFORE the per-axis lineup so an + /// over-declared-arity subscript (whose trailing indices are + /// `MismatchedIterated` from a missing source axis) is a `Mismatch`, not + /// mislabelled by the per-axis arms. + fn derive_other_dep_verdict( + axes: &[OccurrenceAxis], + dep_arity: Option, + target_iterated_count: usize, + ) -> DerivedVerdict { + // Precondition (else normal subscript handling): a non-empty subscript, + // no more indices than the target has iterated dims, and every index a + // bare target-iterated-dim name -- i.e. every axis `Iterated` or + // `MismatchedIterated` (a `Pinned`/`Reduced`/`Dynamic` axis is a + // literal, wildcard, or dynamic index). + if axes.is_empty() || axes.len() > target_iterated_count { + return DerivedVerdict::NotIterated; + } + let all_iterated_or_mismatched = axes.iter().all(|a| { + matches!( + a, + OccurrenceAxis::Iterated { .. } | OccurrenceAxis::MismatchedIterated { .. } + ) + }); + if !all_iterated_or_mismatched { + return DerivedVerdict::NotIterated; + } + // The dep's declared arity gates the collapse: un-threadable keeps the + // transform's permissive collapse; a differing arity is a `Mismatch`. + let Some(arity) = dep_arity else { + return DerivedVerdict::Collapse; + }; + if axes.len() != arity { + return DerivedVerdict::Mismatch; + } + if axes + .iter() + .any(|a| matches!(a, OccurrenceAxis::MismatchedIterated { .. })) + { + return DerivedVerdict::Mismatch; + } + DerivedVerdict::Collapse + } + + fn iterated(dim: &str) -> OccurrenceAxis { + OccurrenceAxis::Iterated { + dim: dim.to_string(), + source_dim: dim.to_string(), + } + } + + fn mismatched(dim: &str) -> OccurrenceAxis { + OccurrenceAxis::MismatchedIterated { + dim: dim.to_string(), + } + } + + #[test] + fn other_dep_verdict_rule_covers_every_branch() { + // Natural equal-arity iterated subscript (`arr[D1,D2]` for arr [D1,D2], + // target [D1,D2]): all `Iterated`, arity matches ⇒ Collapse. + assert_eq!( + derive_other_dep_verdict(&[iterated("d1"), iterated("d2")], Some(2), 2), + DerivedVerdict::Collapse, + ); + // Transposed equal-arity (`arr[D2,D1]`): a `MismatchedIterated` axis + // with matching arity ⇒ Mismatch (the GH #526 wrong-element freeze). + assert_eq!( + derive_other_dep_verdict(&[mismatched("d2"), mismatched("d1")], Some(2), 2), + DerivedVerdict::Mismatch, + ); + // Under-arity (corner a): all `Iterated` but fewer indices than the + // dep's declared arity ⇒ Mismatch, NOT the Collapse the all-`Iterated` + // arms alone would give. + assert_eq!( + derive_other_dep_verdict(&[iterated("d1")], Some(2), 2), + DerivedVerdict::Mismatch, + ); + // Over-target-arity (corner b): more indices than the target has + // iterated dims ⇒ NotIterated, NOT the Mismatch the `MismatchedIterated` + // arm alone would give. + assert_eq!( + derive_other_dep_verdict(&[iterated("d1"), mismatched("d1")], Some(2), 1), + DerivedVerdict::NotIterated, + ); + // A non-iterated axis (a `Pinned` literal / `Dynamic` index) anywhere ⇒ + // NotIterated: not an iterated-dim subscript at all. + assert_eq!( + derive_other_dep_verdict( + &[iterated("d1"), OccurrenceAxis::Pinned("young".to_string())], + Some(2), + 2, + ), + DerivedVerdict::NotIterated, + ); + assert_eq!( + derive_other_dep_verdict(&[OccurrenceAxis::Dynamic], Some(1), 1), + DerivedVerdict::NotIterated, + ); + // Un-threadable dep (declared dims unknown) keeps the permissive + // collapse regardless of the per-axis arms. + assert_eq!( + derive_other_dep_verdict(&[iterated("d1"), mismatched("d2")], None, 2), + DerivedVerdict::Collapse, + ); + } + + #[test] + fn under_arity_iterated_subscript_is_mismatch_not_collapse() { + // Corner (a): `arr[D1]` for arr declared [D1,D2], inside an + // A2A-over-[D1,D2] target. The single index lines up with arr's first + // axis, so the occurrence's axes are all-`Iterated` -- yet the dep's + // declared arity is 2, so + // `classify_other_dep_iterated_dim_subscript` returns `Mismatch` + // (ltm_augment.rs:288, `index_dims.len() != dep_dims.len()`), NOT the + // Collapse a rule keyed on the per-axis arms alone would derive. + // Freezing the wrong element here is the GH #526 silent magnitude error. + let project = TestProject::new("main") + .named_dimension("D1", &["a", "b"]) + .named_dimension("D2", &["x", "y"]) + .array_aux("arr[D1, D2]", "1") + .array_aux_direct( + "t", + vec!["D1".to_string(), "D2".to_string()], + "arr[D1] * 2", + None, + ); + with_ir_and_occ(&project, "t", |_ir, occs| { + let arr = var_occs(occs, "arr"); + assert_eq!(arr.len(), 1, "occs: {occs:?}"); + assert_eq!( + arr[0].shape, + RefShape::DynamicIndex, + "coarse shape unchanged" + ); + assert_eq!( + arr[0].axes, + vec![iterated("d1")], + "the single lined-up index is `Iterated`" + ); + // dep arity 2, target iterated-dim count 2. + assert_eq!( + derive_other_dep_verdict(&arr[0].axes, Some(2), 2), + DerivedVerdict::Mismatch, + "under-arity must derive Mismatch, not Collapse" + ); + }); + } + + #[test] + fn over_target_arity_iterated_subscript_is_not_iterated() { + // Corner (b): `arr[D1,D1]` for arr declared [D1,D2], inside an + // A2A-over-[D1] target. Position 0 lines up (`Iterated`); position 1's + // `D1` names the source's `D2` axis, so it is `MismatchedIterated`. + // But the subscript has MORE indices (2) than the target has iterated + // dims (1), so `classify_other_dep_iterated_dim_subscript` short-circuits + // to `NotIterated` (ltm_augment.rs:268, normal wrap), NOT the Mismatch + // the `MismatchedIterated` arm alone would derive. + let project = TestProject::new("main") + .named_dimension("D1", &["a", "b"]) + .named_dimension("D2", &["x", "y"]) + .array_aux("arr[D1, D2]", "1") + .array_aux_direct("t", vec!["D1".to_string()], "arr[D1, D1] * 2", None); + with_ir_and_occ(&project, "t", |_ir, occs| { + let arr = var_occs(occs, "arr"); + assert_eq!(arr.len(), 1, "occs: {occs:?}"); + assert_eq!( + arr[0].shape, + RefShape::DynamicIndex, + "coarse shape unchanged" + ); + assert_eq!( + arr[0].axes, + vec![iterated("d1"), mismatched("d1")], + "position 1's D1 names the source's D2 axis: MismatchedIterated" + ); + // dep arity 2, target iterated-dim count 1. + assert_eq!( + derive_other_dep_verdict(&arr[0].axes, Some(2), 1), + DerivedVerdict::NotIterated, + "an over-target-arity subscript must derive NotIterated, not Mismatch" + ); + }); + } +} diff --git a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs index fa5c5f730..2aa1391e9 100644 --- a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs +++ b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs @@ -186,9 +186,13 @@ fn expr0_routes_through_agg(name: &str, arity: usize) -> bool { /// `collect_all_reference_sites`: /// /// * a bare `Var` head is a `Bare` occurrence; -/// * a `Subscript` head records its indices, THEN recurses into them (so an -/// index-nested model variable -- `other[from]` -- is recorded too, exactly -/// as the Expr2 walker recurses index expressions); +/// * a `Subscript` head records its indices, THEN recurses into the +/// non-element-selector ones (so an index-nested model variable -- +/// `other[from]` -- is recorded too, exactly as the Expr2 walker recurses +/// index expressions). An index that resolves to a literal element of the +/// subscripted variable's axis is an element SELECTOR, not a causal +/// reference, and is skipped on BOTH families (the A2a element-vs-variable +/// collision resolution -- see the `Subscript` arm below); /// * a reducer `App` makes its arguments `in_reducer` (sticky through nested /// reducers), matching the Expr2 walker's `reducer_keys` propagation; /// * a `LOOKUP` call's first (table) argument is skipped, matching the Expr2 @@ -224,7 +228,27 @@ fn collect_expr0_occurrences( in_reducer, }); } - for idx in indices { + // Mirror the Expr2 walker's element-selector skip: an index that + // resolves to a literal element of the subscripted variable's axis + // is an element SELECTOR (execution resolves it to a static offset, + // element taking priority over any like-named variable), not a + // causal reference -- so it is NOT an occurrence, even when it + // collides with a like-named variable (`arr[nyc]` with a variable + // `nyc`). This uses the Expr0 sibling resolver + // (`resolve_literal_element_index`), so this side of the gate skips + // exactly the token the production transform + // (`wrap_index_non_matching_in_previous`) leaves verbatim -- both + // classifier families agree it is not a site (the A2a element-vs- + // variable-collision resolution). + let source_dim_elements: Vec> = + source_dims_of(variables, canonical.as_str()) + .iter() + .map(dimension_element_names) + .collect(); + for (i, idx) in indices.iter().enumerate() { + if resolve_literal_element_index(idx, i, &source_dim_elements).is_some() { + continue; + } match idx { IndexExpr0::Expr(e) => collect_expr0_occurrences(e, variables, in_reducer, out), IndexExpr0::Range(l, r, _) => { @@ -775,16 +799,34 @@ fn agree_dimension_name_index_gh759() { #[test] fn agree_element_variable_name_collision() { // A model variable named identically to a dimension element (`nyc`): the - // subscript `population[nyc]` is a FixedIndex element reference while the - // bare `nyc` is a variable reference. Both families must keep them - // distinct (element in subscript, variable outside). + // subscript `population[nyc]` selects the ELEMENT (execution resolves it to + // a static offset, element taking priority over the like-named variable), + // while the bare `nyc` outside the subscript is a genuine variable + // reference. Both families must agree: `population -> collide` is a + // FixedIndex site; `nyc -> collide` is ONE Bare site (the bare `* nyc` + // multiplication), NOT two -- the index-nested `nyc` is an element + // selector, not a causal reference. Before the A2a fix the Expr2 walker + // minted an extra `nyc -> collide` Bare site from the subscript index that + // disagreed with the transform; both families now skip it. That site was an + // orphan -- variable-level dep extraction already filtered the element + // token, so no keyed consumer ever read it -- not a consumer-visible edge. let tp = TestProject::new("main") .with_sim_time(0.0, 3.0, 1.0) .named_dimension("Region", &["nyc", "boston"]) .scalar_aux("nyc", "3") .array_aux("population[Region]", "100") .array_aux("collide[Region]", "population[nyc] * nyc"); - assert_classifier_families_agree(&tp); + let compared = assert_classifier_families_agree(&tp); + // Non-vacuity anchor: exactly one Bare `nyc -> collide` site (the bare + // multiplication), not the pre-fix two. If the element-selector skip + // regresses on either family, this pin (or the agreement assertion) fails. + pin_edge(&compared, "nyc", "collide", &[RefShape::Bare]); + pin_edge( + &compared, + "population", + "collide", + &[RefShape::FixedIndex(vec!["nyc".to_string()])], + ); } #[test] From f057ef3828f545f641d28e72ef0a0e28a6f76d9b Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sat, 18 Jul 2026 09:29:22 -0700 Subject: [PATCH 02/14] engine: pick module-output live dep in document order for link scores module_link_score_equation's module-to-variable branch holds one output of the module instance live in the ceteris-paribus partial and freezes the rest. The live output was chosen by .find() over identifier_set's HashSet, whose iteration order is per-process random, so when a target reads two outputs of one module instance the emitted link score -- and every loop score through it -- flapped between processes (GH #971). The reference-site IR now enumerates module-output composites as per-occurrence ModuleOutput records in stable left-to-right walk order, so the pick becomes the document-order-first output that names the module: deterministic across processes, over the same candidate set the old scan considered. The identifier_set scan survives only as a defensive fallback when the IR recorded no matching occurrence, preserving the historical set of edges that receive a real partial; the typed-equation work is expected to delete it. The pin builds two models differing only in the order the target reads the two outputs and asserts they hold different sources live, distinguishing document order from both an alphabetical sort and the old order-independent hash pick. --- src/simlin-engine/src/db.rs | 69 +++++++++-- src/simlin-engine/src/db/ltm_module_tests.rs | 121 +++++++++++++++++++ 2 files changed, 181 insertions(+), 9 deletions(-) diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index 7d5ae0696..a98823c16 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -660,6 +660,43 @@ fn module_composite_ports( .collect() } +/// The first `{module}·port` output composite that `to`'s equation reads, in +/// DOCUMENT order (left-to-right over `to`'s reconstructed AST), or `None` +/// when `to` reads no output of `module`. +/// +/// This is the deterministic replacement (GH #971) for +/// [`module_link_score_equation`]'s former pick -- an arbitrary +/// `{module}·`-prefixed dependency out of `identifier_set`'s `HashSet`, whose +/// iteration order is per-process random. When `to` reads MORE THAN ONE +/// output of one module instance the choice decides which output's change the +/// link score attributes (the others are frozen), so the random pick made the +/// emitted score -- and every loop score through it -- flap between processes. +/// The reference-site IR already enumerates the module-output composites as +/// `OccurrenceRef::ModuleOutput` occurrences in the walk's stable +/// left-to-right order, so taking the first that names `module` is a +/// reproducible choice over the SAME set the old scan considered. +fn module_output_ref_in_document_order( + db: &dyn Db, + model: SourceModel, + project: SourceProject, + module: &str, + to_name: &str, +) -> Option { + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + ref_sites + .occurrences + .get(to_name)? + .iter() + .find_map(|occ| match &occ.reference { + crate::db::ltm_ir::OccurrenceRef::ModuleOutput { + module: occ_module, + composite, + .. + } if occ_module == module => Some(composite.clone()), + _ => None, + }) +} + /// Equation for a module-involved link score (`from` and/or `to` is a /// module node in the parent causal graph). Shared verbatim by the /// `(from, to)`-keyed [`link_score_equation_text`] and the per-shape @@ -805,15 +842,29 @@ pub(crate) fn module_link_score_equation( // module output via `module·port`. Prefer a ceteris-paribus // partial on that equation (the exact link score); fall back to // the unit transfer if the reference can't be located. - let from_output = to_var - .ast() - .map(|ast| crate::variable::identifier_set(ast, &[], None)) - .and_then(|deps| { - let prefix = format!("{from_name}\u{00B7}"); - deps.into_iter() - .find(|d| d.as_str().starts_with(&prefix)) - .map(|d| d.to_string()) - }); + // + // Which output to hold live (GH #971): the reference-site IR + // enumerates the module-output composites in DOCUMENT order, so take + // the first that names `from` -- a deterministic pick across + // processes, unlike the former `identifier_set().find()` over a + // per-process-random `HashSet`. The `identifier_set` scan is kept + // only as a defensive fallback for the (unexpected) case where the IR + // recorded no matching `ModuleOutput` occurrence for `to`, preserving + // the historical set of edges that receive a real partial. + let from_output = module_output_ref_in_document_order( + db, model, project, from_name, to_name, + ) + .or_else(|| { + to_var + .ast() + .map(|ast| crate::variable::identifier_set(ast, &[], None)) + .and_then(|deps| { + let prefix = format!("{from_name}\u{00B7}"); + deps.into_iter() + .find(|d| d.as_str().starts_with(&prefix)) + .map(|d| d.to_string()) + }) + }); match from_output { Some(output_ref) => { let output_ident = Ident::::new(&output_ref); diff --git a/src/simlin-engine/src/db/ltm_module_tests.rs b/src/simlin-engine/src/db/ltm_module_tests.rs index c53998cc0..756d6e772 100644 --- a/src/simlin-engine/src/db/ltm_module_tests.rs +++ b/src/simlin-engine/src/db/ltm_module_tests.rs @@ -1738,3 +1738,124 @@ fn test_pinned_loop_through_stockless_passthrough_still_rejected() { "rejection must carry the clear no-stock reason; got: {reason}" ); } + +/// GH #971: when a target reads MORE THAN ONE output of a single module +/// instance, `module_link_score_equation`'s module->variable branch holds ONE +/// output live (its change is what the score attributes) and freezes the +/// rest. The live output was formerly chosen by an arbitrary `.find()` over +/// `identifier_set`'s per-process-random `HashSet`, so the emitted score -- +/// and every loop score through it -- flapped between processes. The fix +/// selects the DOCUMENT-order-first output from the reference-site IR's +/// per-occurrence enumeration (`OccurrenceRef::ModuleOutput`, walk order). +/// +/// This pins the fix by building two models that differ ONLY in the order the +/// target reads the module's two outputs and asserting they hold DIFFERENT +/// sources live. That distinguishes document order from BOTH stable-but-wrong +/// alternatives: an alphabetical sort would pick `out_a` in both, and the old +/// hash order could not depend on reading order at all. The live source is +/// identifiable because it -- and only it -- drives the guard form's SIGN +/// normalizer `(src - PREVIOUS(src))`; the frozen output appears solely inside +/// a `PREVIOUS(...)`. +#[test] +fn test_multi_output_module_link_score_holds_document_order_first_live() { + use salsa::Setter; + + let build_link_eqn = |combined_eqn: &str| -> String { + let project = datamodel::Project { + name: "multi_output".to_string(), + sim_specs: datamodel::SimSpecs { + start: 0.0, + stop: 10.0, + dt: datamodel::Dt::Dt(1.0), + save_step: None, + sim_method: datamodel::SimMethod::Euler, + time_units: None, + }, + dimensions: vec![], + units: vec![], + models: vec![ + x_model( + "main", + vec![ + x_stock("level", "50", &["adjustment"], &[], None), + x_aux("combined", combined_eqn, None), + x_flow("adjustment", "combined / 5", None), + x_module("multi_out", &[("level", "multi_out.input")], None), + ], + ), + x_model( + "multi_out", + vec![ + datamodel::Variable::Aux(datamodel::Aux { + ident: "input".to_string(), + equation: datamodel::Equation::Scalar("0".to_string()), + documentation: String::new(), + units: None, + gf: None, + ai_state: None, + uid: None, + compat: datamodel::Compat { + can_be_module_input: true, + ..datamodel::Compat::default() + }, + }), + x_aux("out_a", "input * 2", None), + x_aux("out_b", "input * 3", None), + ], + ), + ], + source: None, + ai_information: None, + }; + + let mut db = SimlinDb::default(); + let (source_project, main_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + + let ltm_vars = model_ltm_variables(&db, main_model, source_project); + let link = ltm_vars + .vars + .iter() + .find(|v| v.name.contains("multi_out\u{2192}combined")) + .unwrap_or_else(|| { + panic!( + "must emit the multi_out->combined link score; got: {:?}", + ltm_vars.vars.iter().map(|v| &v.name).collect::>() + ) + }); + match &link.equation { + datamodel::Equation::Scalar(text) => text.clone(), + other => panic!("module link score should be scalar, got {other:?}"), + } + }; + + // The live source is the sole reference spelled as `(src - PREVIOUS(src))` + // (the SIGN normalizer); a frozen output only ever appears wrapped in + // `PREVIOUS(...)`. + let live_diff = + |out: &str| format!("\"multi_out\u{00B7}{out}\" - PREVIOUS(\"multi_out\u{00B7}{out}\")"); + + let eqn_ab = build_link_eqn("multi_out.out_a + multi_out.out_b"); + assert!( + eqn_ab.contains(&live_diff("out_a")), + "reads out_a first, so out_a is held live: {eqn_ab}" + ); + assert!( + !eqn_ab.contains(&live_diff("out_b")), + "out_b (read second) is frozen, never the SIGN source: {eqn_ab}" + ); + + let eqn_ba = build_link_eqn("multi_out.out_b + multi_out.out_a"); + assert!( + eqn_ba.contains(&live_diff("out_b")), + "reads out_b first, so out_b is held live -- document order, not \ + alphabetical (which would pick out_a): {eqn_ba}" + ); + assert!( + !eqn_ba.contains(&live_diff("out_a")), + "out_a (read second) is frozen: {eqn_ba}" + ); +} From b7898692738d201cdc1cb9ff99e5f41ca27ec5fc Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sat, 18 Jul 2026 14:01:44 -0700 Subject: [PATCH 03/14] engine: run LTM ceteris-paribus wraps on original equations The ceteris-paribus transform was applied to three categories of input text, two of them DERIVED: generate_per_element_link_equation wrapped a row-rewritten equation against a synthesized FixedIndex(row) live shape, and the agg-to-scalar / scalar-to-element generators wrapped agg-substituted text where hoisted reducers had already been replaced by synthetic names. Derived text has no reference-site occurrence stream, which blocked moving live-ref selection onto the per-occurrence IR: a partial switch would have left three classification paths (see the Category A/B/C analysis recorded in the track-A PR). This inverts the composition to rewrite-of-a-transform: every wrap now runs on the target's own equation (or its own Arrayed slot text) with the site's actual shape as the live shape, and the row-pinning and reducer-substitution rewrites become free-standing post-transform lowerings (rewrite_per_element_source_refs on the wrapped AST, substitute_reducers_in_expr0 mapping frozen PREVIOUS(SUM(..)) and live reducer subexpressions to their synthetic agg names in place). The lowerings live in a new sibling module, ltm_augment_post_transform.rs, both because they are the coherent post-wrap unit and to keep ltm_augment.rs under the per-file line cap. Net new Expr0 classification call sites: zero -- the follow-on step can swap the classifier family for occurrence lookups wholesale. Behavior is pinned by a new characterization suite (ltm_char_tests.rs with checked-in goldens plus runtime guards): generated texts are byte-identical to the prior composition for the entire battery except one adjudicated case -- a frozen occurrence reached through another reference's subscript index is now element-qualified (other[pop[region-dot-boston, ..]]) where the old ordering left the bare element spelling; both resolve to the same element and the full compiled series is bitwise identical. Guards also deliberately pin two pre-existing degradations (a first-live-step NaN from an uninitialized PREVIOUS(index) capture helper, and loud warned-zero on nested live reducers) so this change provably preserves rather than fixes them; they are tracked separately. --- src/simlin-engine/src/db.rs | 2 + src/simlin-engine/src/db/ltm/link_scores.rs | 92 +- .../db/ltm_char_golden/agg_nested_reducer.txt | 10 + .../ltm_char_golden/agg_to_arrayed_target.txt | 4 + .../ltm_char_golden/agg_to_scalar_target.txt | 2 + .../ltm_char_golden/arrayed_agg_to_target.txt | 14 + .../arrayed_target_slot_scores.txt | 10 + .../per_element_ambiguous_pin.txt | 10 + .../per_element_dynamic_index.txt | 6 + .../per_element_index_nested_occurrence.txt | 4 + .../per_element_link_scores.txt | 4 + .../per_element_mapped_occurrence.txt | 10 + .../per_element_mixed_occurrences.txt | 10 + .../per_element_other_deps.txt | 8 + src/simlin-engine/src/db/ltm_char_tests.rs | 1120 +++++++++++++++++ src/simlin-engine/src/ltm_augment.rs | 1009 ++++++--------- .../src/ltm_augment_post_transform.rs | 455 +++++++ 17 files changed, 2110 insertions(+), 660 deletions(-) create mode 100644 src/simlin-engine/src/db/ltm_char_golden/agg_nested_reducer.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/agg_to_arrayed_target.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/agg_to_scalar_target.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/arrayed_agg_to_target.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/arrayed_target_slot_scores.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_ambiguous_pin.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_dynamic_index.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_index_nested_occurrence.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_link_scores.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_mapped_occurrence.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_mixed_occurrences.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/per_element_other_deps.txt create mode 100644 src/simlin-engine/src/db/ltm_char_tests.rs create mode 100644 src/simlin-engine/src/ltm_augment_post_transform.rs diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index a98823c16..762512702 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -1432,6 +1432,8 @@ mod fragment_cache_tests; #[cfg(test)] mod incremental_compile_tests; #[cfg(test)] +mod ltm_char_tests; +#[cfg(test)] mod ltm_module_tests; #[cfg(test)] mod ltm_unified_tests; diff --git a/src/simlin-engine/src/db/ltm/link_scores.rs b/src/simlin-engine/src/db/ltm/link_scores.rs index 186460ba7..684d3ecfa 100644 --- a/src/simlin-engine/src/db/ltm/link_scores.rs +++ b/src/simlin-engine/src/db/ltm/link_scores.rs @@ -1225,6 +1225,10 @@ pub(super) fn try_scalar_to_arrayed_link_scores( // LoadPrev, no helper auxes); the NAME below keeps the bare form. &crate::ltm_augment::qualify_element_csv(element, &to_dims), elem_text, + // A true scalar source has no hoisted reducers, so the wrap + // holds `from` live as an ident and the post-transform + // substitution is a no-op. + &std::collections::HashMap::new(), elem_deps, deps_to_sub, // A true scalar source: the bare `quote_ident(from)` @@ -3316,19 +3320,12 @@ pub(super) fn emit_agg_to_target_link_scores( .collect() }; - // Helper: substitute the reducers in a slot expr's canonical text, to - // build the agg→target link-score equation for one target element (or the - // scalar case when `element` is `None`). Propagates a `PartialEquationError` - // when the reducer substitution can't parse its input -- the caller skips - // the variable and warns rather than emitting a partial that keeps the - // inline reducer live instead of the hoisted agg node (GH #661). - let slot_text = - |expr: &crate::ast::Expr2| -> Result { - crate::ltm_augment::substitute_reducers_in_equation( - &crate::patch::expr2_to_string(expr), - &reducer_subst, - ) - }; + // Track A stage 1: reducer -> agg-name substitution is now a POST-transform + // lowering INSIDE the generators (`generate_agg_to_scalar_target_equation` / + // `generate_scalar_to_element_equation`), which run the wrap on `to`'s OWN + // equation and then substitute. Each branch therefore threads the + // un-substituted slot text + `reducer_subst` rather than pre-substituting + // here (the old caller-side `slot_text` helper). match ast { Ast::Scalar(expr) => { @@ -3340,19 +3337,16 @@ pub(super) fn emit_agg_to_target_link_scores( "$\u{205A}ltm\u{205A}link_score\u{205A}{}\u{2192}{}", agg.name, to ); - let substituted = match slot_text(expr) { - Ok(substituted) => substituted, - Err(err) => { - if unscoreable_edges.insert((agg.name.clone(), to.to_string())) { - emit_ltm_partial_equation_warning(db, model, &name, &err); - } - return; - } - }; + // Track A stage 1: the generator now runs the wrap on `to`'s OWN + // equation and substitutes the reducers -> agg names AFTER the wrap, + // so we pass the un-substituted text + `reducer_subst` (the prior + // caller-side `slot_text`/`substitute_reducers_in_equation` step and + // its parse-failure handling move inside the generator's `Err`). match crate::ltm_augment::generate_agg_to_scalar_target_equation( &agg.name, to, - &substituted, + &crate::patch::expr2_to_string(expr), + &reducer_subst, &all_deps, Some(project_dimensions_context(db, project)), // GH #910: a scalar with-lookup target's single table, @@ -3374,24 +3368,14 @@ pub(super) fn emit_agg_to_target_link_scores( } } Ast::ApplyToAll(_, expr) => { - // One shared body; emit one per-target-element scalar var. A - // substitution parse failure here fails the whole edge (the body - // is shared across every element), so warn once on the base - // `agg → to` name and skip rather than emit a partial that keeps - // the inline reducer live (GH #661). - let substituted = match slot_text(expr) { - Ok(substituted) => substituted, - Err(err) => { - let name = format!( - "$\u{205A}ltm\u{205A}link_score\u{205A}{}\u{2192}{}", - agg.name, to - ); - if unscoreable_edges.insert((agg.name.clone(), to.to_string())) { - emit_ltm_partial_equation_warning(db, model, &name, &err); - } - return; - } - }; + // One shared body; emit one per-target-element scalar var. Track A + // stage 1: the generator runs the wrap on this OWN A2A body and + // substitutes reducers -> agg names AFTER the wrap, so we thread the + // un-substituted text + `reducer_subst`. An own-text parse failure + // (unreachable -- the text is a `print_eqn` of a compiled AST) then + // surfaces on the first element's generator `Err` and dooms the edge + // once, via the same GH #661 warning path the per-element loop uses. + let to_own_text = crate::patch::expr2_to_string(expr); if to_dims.is_empty() { return; } @@ -3421,7 +3405,8 @@ pub(super) fn emit_agg_to_target_link_scores( to, // Qualified for equation text; the name keeps the bare form. &crate::ltm_augment::qualify_element_csv(element, &to_dims), - &substituted, + &to_own_text, + &reducer_subst, &all_deps, &deps_to_subscript, Some(&agg_source_ref_for_target(element)), @@ -3466,16 +3451,20 @@ pub(super) fn emit_agg_to_target_link_scores( // iff there is no slot expression. let equation = match per_elem.get(&canonical_elem).or(default_expr.as_ref()) { None => Ok("0".to_string()), - // A substitution parse failure rides the same `Result` the - // equation builder returns, so the `match equation` below - // converts it into the per-element `Warning` + skip (GH - // #661) -- no separate error handling needed here. - Some(slot_expr) => slot_text(slot_expr).and_then(|substituted| { + // Track A stage 1: the generator runs the wrap on this OWN + // slot text and substitutes the reducers -> agg names AFTER + // the wrap, so we thread the un-substituted slot text + + // `reducer_subst`. An own-text parse failure rides the same + // `Result` the equation builder returns, so the `match + // equation` below converts it into the per-element `Warning` + // + skip (GH #661) -- no separate error handling needed here. + Some(slot_expr) => { // Re-derive per-slot deps (the union over all slots // would over-freeze refs absent from this slot), // then extend with this agg's name and every other - // agg referenced in the (substituted) slot text so - // they are all PREVIOUS-wrapped (ceteris paribus). + // agg name so they are all PREVIOUS-wrapped (ceteris + // paribus); the reducer-source vars are already in the + // own-slot deps. let mut slot_deps = crate::variable::classify_dependencies( &Ast::Scalar(slot_expr.clone()), target_ast_dims, @@ -3491,7 +3480,8 @@ pub(super) fn emit_agg_to_target_link_scores( to, // Qualified for equation text; the name keeps the bare form. &crate::ltm_augment::qualify_element_csv(element, &to_dims), - &substituted, + &crate::patch::expr2_to_string(slot_expr), + &reducer_subst, &slot_deps, &deps_to_subscript, Some(&agg_source_ref_for_target(element)), @@ -3499,7 +3489,7 @@ pub(super) fn emit_agg_to_target_link_scores( Some(project_dimensions_context(db, project)), elem_gf_ref(element).as_deref(), ) - }), + } }; let name = format!( "$\u{205A}ltm\u{205A}link_score\u{205A}{}\u{2192}{}[{}]", diff --git a/src/simlin-engine/src/db/ltm_char_golden/agg_nested_reducer.txt b/src/simlin-engine/src/db/ltm_char_golden/agg_nested_reducer.txt new file mode 100644 index 000000000..9c68dce6b --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/agg_nested_reducer.txt @@ -0,0 +1,10 @@ +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0→growth[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston] - PREVIOUS(growth[region·boston])) = 0) OR (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) then 0 else SAFEDIV(((sum(previous(matrix[region·boston, *]) * previous(other[region·nyc, *]) * "$⁚ltm⁚agg⁚0") * 0.001) - PREVIOUS(growth[region·boston])), ABS((growth[region·boston] - PREVIOUS(growth[region·boston]))), 0) * SIGN(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))) +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0→growth[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc] - PREVIOUS(growth[region·nyc])) = 0) OR (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) then 0 else SAFEDIV(((sum(previous(matrix[region·nyc, *]) * previous(other[region·nyc, *]) * "$⁚ltm⁚agg⁚0") * 0.001) - PREVIOUS(growth[region·nyc])), ABS((growth[region·nyc] - PREVIOUS(growth[region·nyc]))), 0) * SIGN(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))) +$⁚ltm⁚link_score⁚growth→stock dims=[Region] +a2a[Region]: if (TIME = INITIAL_TIME) OR (PREVIOUS(TIME, INITIAL_TIME) = INITIAL_TIME) then 0 else ABS(SAFEDIV((time_step * (PREVIOUS(growth[Region]) - PREVIOUS(PREVIOUS(growth[Region])))), ((stock[Region] - PREVIOUS(stock[Region])) - (PREVIOUS(stock[Region]) - PREVIOUS(PREVIOUS(stock[Region])))), 0)) +$⁚ltm⁚link_score⁚pop[boston]→$⁚ltm⁚agg⁚0 +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0") + (pop[region·boston] - PREVIOUS(pop[region·boston])) - PREVIOUS("$⁚ltm⁚agg⁚0")), ABS(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) +$⁚ltm⁚link_score⁚pop[nyc]→$⁚ltm⁚agg⁚0 +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0") + (pop[region·nyc] - PREVIOUS(pop[region·nyc])) - PREVIOUS("$⁚ltm⁚agg⁚0")), ABS(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/agg_to_arrayed_target.txt b/src/simlin-engine/src/db/ltm_char_golden/agg_to_arrayed_target.txt new file mode 100644 index 000000000..7f9a6e17b --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/agg_to_arrayed_target.txt @@ -0,0 +1,4 @@ +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0→outflow[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((outflow[region·boston] - PREVIOUS(outflow[region·boston])) = 0) OR (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) then 0 else SAFEDIV((("$⁚ltm⁚agg⁚0" * previous(frac[region·boston])) - PREVIOUS(outflow[region·boston])), ABS((outflow[region·boston] - PREVIOUS(outflow[region·boston]))), 0) * SIGN(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))) +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0→outflow[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((outflow[region·nyc] - PREVIOUS(outflow[region·nyc])) = 0) OR (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) then 0 else SAFEDIV((("$⁚ltm⁚agg⁚0" * previous(frac[region·nyc])) - PREVIOUS(outflow[region·nyc])), ABS((outflow[region·nyc] - PREVIOUS(outflow[region·nyc]))), 0) * SIGN(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/agg_to_scalar_target.txt b/src/simlin-engine/src/db/ltm_char_golden/agg_to_scalar_target.txt new file mode 100644 index 000000000..429bfbe43 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/agg_to_scalar_target.txt @@ -0,0 +1,2 @@ +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0→total +scalar: if (TIME = INITIAL_TIME) then 0 else if ((total - PREVIOUS(total)) = 0) OR (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) then 0 else SAFEDIV((("$⁚ltm⁚agg⁚0" * 2) - PREVIOUS(total)), ABS((total - PREVIOUS(total))), 0) * SIGN(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/arrayed_agg_to_target.txt b/src/simlin-engine/src/db/ltm_char_golden/arrayed_agg_to_target.txt new file mode 100644 index 000000000..e21230e96 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/arrayed_agg_to_target.txt @@ -0,0 +1,14 @@ +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0[boston]→outflow[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((outflow[region·boston] - PREVIOUS(outflow[region·boston])) = 0) OR (("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston])) = 0) then 0 else SAFEDIV((("$⁚ltm⁚agg⁚0"[region·boston] * 0.1) - PREVIOUS(outflow[region·boston])), ABS((outflow[region·boston] - PREVIOUS(outflow[region·boston]))), 0) * SIGN(("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston]))) +$⁚ltm⁚link_score⁚$⁚ltm⁚agg⁚0[nyc]→outflow[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((outflow[region·nyc] - PREVIOUS(outflow[region·nyc])) = 0) OR (("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc])) = 0) then 0 else SAFEDIV((("$⁚ltm⁚agg⁚0"[region·nyc] * 0.1) - PREVIOUS(outflow[region·nyc])), ABS((outflow[region·nyc] - PREVIOUS(outflow[region·nyc]))), 0) * SIGN(("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc]))) +$⁚ltm⁚link_score⁚matrix[boston,a]→$⁚ltm⁚agg⁚0[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston])) = 0) OR ((matrix[region·boston,dim2·a] - PREVIOUS(matrix[region·boston,dim2·a])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0"[boston]) + (matrix[region·boston,dim2·a] - PREVIOUS(matrix[region·boston,dim2·a])) - PREVIOUS("$⁚ltm⁚agg⁚0"[boston])), ABS(("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston]))), 0) * SIGN((matrix[region·boston,dim2·a] - PREVIOUS(matrix[region·boston,dim2·a]))) +$⁚ltm⁚link_score⁚matrix[boston,b]→$⁚ltm⁚agg⁚0[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston])) = 0) OR ((matrix[region·boston,dim2·b] - PREVIOUS(matrix[region·boston,dim2·b])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0"[boston]) + (matrix[region·boston,dim2·b] - PREVIOUS(matrix[region·boston,dim2·b])) - PREVIOUS("$⁚ltm⁚agg⁚0"[boston])), ABS(("$⁚ltm⁚agg⁚0"[boston] - PREVIOUS("$⁚ltm⁚agg⁚0"[boston]))), 0) * SIGN((matrix[region·boston,dim2·b] - PREVIOUS(matrix[region·boston,dim2·b]))) +$⁚ltm⁚link_score⁚matrix[nyc,a]→$⁚ltm⁚agg⁚0[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc])) = 0) OR ((matrix[region·nyc,dim2·a] - PREVIOUS(matrix[region·nyc,dim2·a])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0"[nyc]) + (matrix[region·nyc,dim2·a] - PREVIOUS(matrix[region·nyc,dim2·a])) - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc])), ABS(("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc]))), 0) * SIGN((matrix[region·nyc,dim2·a] - PREVIOUS(matrix[region·nyc,dim2·a]))) +$⁚ltm⁚link_score⁚matrix[nyc,b]→$⁚ltm⁚agg⁚0[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc])) = 0) OR ((matrix[region·nyc,dim2·b] - PREVIOUS(matrix[region·nyc,dim2·b])) = 0) then 0 else SAFEDIV((PREVIOUS("$⁚ltm⁚agg⁚0"[nyc]) + (matrix[region·nyc,dim2·b] - PREVIOUS(matrix[region·nyc,dim2·b])) - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc])), ABS(("$⁚ltm⁚agg⁚0"[nyc] - PREVIOUS("$⁚ltm⁚agg⁚0"[nyc]))), 0) * SIGN((matrix[region·nyc,dim2·b] - PREVIOUS(matrix[region·nyc,dim2·b]))) +$⁚ltm⁚link_score⁚outflow→stock dims=[Region] +a2a[Region]: if (TIME = INITIAL_TIME) OR (PREVIOUS(TIME, INITIAL_TIME) = INITIAL_TIME) then 0 else ABS(SAFEDIV((time_step * (PREVIOUS(outflow[Region]) - PREVIOUS(PREVIOUS(outflow[Region])))), ((stock[Region] - PREVIOUS(stock[Region])) - (PREVIOUS(stock[Region]) - PREVIOUS(PREVIOUS(stock[Region])))), 0)) diff --git a/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_slot_scores.txt b/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_slot_scores.txt new file mode 100644 index 000000000..529087691 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/arrayed_target_slot_scores.txt @@ -0,0 +1,10 @@ +$⁚ltm⁚link_score⁚pop[boston]→mp dims=[Region] +arrayed[Region] (apply_default=true): + boston => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV((((pop[boston] - PREVIOUS(pop[region·nyc])) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) + nyc => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV((((PREVIOUS(pop[region·nyc]) - pop[boston]) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) + => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·boston] - PREVIOUS(pop[region·boston])) = 0) then 0 else SAFEDIV(((0) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·boston] - PREVIOUS(pop[region·boston]))) +$⁚ltm⁚link_score⁚pop[nyc]→mp dims=[Region] +arrayed[Region] (apply_default=true): + boston => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV((((PREVIOUS(pop[region·boston]) - pop[nyc]) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) + nyc => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV((((pop[nyc] - PREVIOUS(pop[region·boston])) * 0.01) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) + => if (TIME = INITIAL_TIME) then 0 else if ((mp - PREVIOUS(mp)) = 0) OR ((pop[region·nyc] - PREVIOUS(pop[region·nyc])) = 0) then 0 else SAFEDIV(((0) - PREVIOUS(mp)), ABS((mp - PREVIOUS(mp))), 0) * SIGN((pop[region·nyc] - PREVIOUS(pop[region·nyc]))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_ambiguous_pin.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_ambiguous_pin.txt new file mode 100644 index 000000000..8a0bf294c --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_ambiguous_pin.txt @@ -0,0 +1,10 @@ +$⁚ltm⁚link_score⁚pop[boston,old]→growth dims=[Region,Age] +a2a[Region,Age]: if (TIME = INITIAL_TIME) then 0 else if ((growth - PREVIOUS(growth)) = 0) OR ((pop[region·boston,age·old] - PREVIOUS(pop[region·boston,age·old])) = 0) then 0 else SAFEDIV(((PREVIOUS(pop[region, age·young]) * pop[boston, old]) - PREVIOUS(growth)), ABS((growth - PREVIOUS(growth))), 0) * SIGN((pop[region·boston,age·old] - PREVIOUS(pop[region·boston,age·old]))) +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston,age·old] - PREVIOUS(growth[region·boston,age·old])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·old])) - PREVIOUS(growth[region·boston,age·old])), ABS((growth[region·boston,age·old] - PREVIOUS(growth[region·boston,age·old]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston,age·young] - PREVIOUS(growth[region·boston,age·young])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·old])) - PREVIOUS(growth[region·boston,age·young])), ABS((growth[region·boston,age·young] - PREVIOUS(growth[region·boston,age·young]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc,age·old] - PREVIOUS(growth[region·nyc,age·old])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·boston, age·old])) - PREVIOUS(growth[region·nyc,age·old])), ABS((growth[region·nyc,age·old] - PREVIOUS(growth[region·nyc,age·old]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc,age·young] - PREVIOUS(growth[region·nyc,age·young])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·boston, age·old])) - PREVIOUS(growth[region·nyc,age·young])), ABS((growth[region·nyc,age·young] - PREVIOUS(growth[region·nyc,age·young]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_dynamic_index.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_dynamic_index.txt new file mode 100644 index 000000000..175d7a9eb --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_dynamic_index.txt @@ -0,0 +1,6 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston] - PREVIOUS(growth[region·boston])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, PREVIOUS(idx)]) * 0.001) - PREVIOUS(growth[region·boston])), ABS((growth[region·boston] - PREVIOUS(growth[region·boston]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc] - PREVIOUS(growth[region·nyc])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·nyc, PREVIOUS(idx)]) * 0.001) - PREVIOUS(growth[region·nyc])), ABS((growth[region·nyc] - PREVIOUS(growth[region·nyc]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop→growth dims=[Region] +a2a[Region]: if (TIME = INITIAL_TIME) then 0 else if ((growth - PREVIOUS(growth)) = 0) OR ((SUM(pop[region, PREVIOUS(idx)]) - PREVIOUS(SUM(pop[region, PREVIOUS(idx)]))) = 0) then 0 else SAFEDIV(((PREVIOUS(pop[region, age·young]) * pop[region, PREVIOUS(idx)] * 0.001) - PREVIOUS(growth)), ABS((growth - PREVIOUS(growth))), 0) * SIGN((SUM(pop[region, PREVIOUS(idx)]) - PREVIOUS(SUM(pop[region, PREVIOUS(idx)])))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_index_nested_occurrence.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_index_nested_occurrence.txt new file mode 100644 index 000000000..824b1e9e3 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_index_nested_occurrence.txt @@ -0,0 +1,4 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston] - PREVIOUS(growth[region·boston])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * previous(other[pop[region·boston, age·young]])) - PREVIOUS(growth[region·boston])), ABS((growth[region·boston] - PREVIOUS(growth[region·boston]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc] - PREVIOUS(growth[region·nyc])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * previous(other[pop[region·nyc, age·young]])) - PREVIOUS(growth[region·nyc])), ABS((growth[region·nyc] - PREVIOUS(growth[region·nyc]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_link_scores.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_link_scores.txt new file mode 100644 index 000000000..30bb08123 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_link_scores.txt @@ -0,0 +1,4 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston] - PREVIOUS(growth[region·boston])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * 0.1) - PREVIOUS(growth[region·boston])), ABS((growth[region·boston] - PREVIOUS(growth[region·boston]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc] - PREVIOUS(growth[region·nyc])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * 0.1) - PREVIOUS(growth[region·nyc])), ABS((growth[region·nyc] - PREVIOUS(growth[region·nyc]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_mapped_occurrence.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_mapped_occurrence.txt new file mode 100644 index 000000000..707f69597 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_mapped_occurrence.txt @@ -0,0 +1,10 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→growth[east,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[state·east,age·old] - PREVIOUS(growth[state·east,age·old])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·old]) * 0.01) - PREVIOUS(growth[state·east,age·old])), ABS((growth[state·east,age·old] - PREVIOUS(growth[state·east,age·old]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[boston,young]→growth[east,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[state·east,age·young] - PREVIOUS(growth[state·east,age·young])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·young]) * 0.01) - PREVIOUS(growth[state·east,age·young])), ABS((growth[state·east,age·young] - PREVIOUS(growth[state·east,age·young]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[west,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[state·west,age·old] - PREVIOUS(growth[state·west,age·old])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·nyc, age·old]) * 0.01) - PREVIOUS(growth[state·west,age·old])), ABS((growth[state·west,age·old] - PREVIOUS(growth[state·west,age·old]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[west,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[state·west,age·young] - PREVIOUS(growth[state·west,age·young])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·nyc, age·young]) * 0.01) - PREVIOUS(growth[state·west,age·young])), ABS((growth[state·west,age·young] - PREVIOUS(growth[state·west,age·young]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop→growth dims=[State,Age] +a2a[State,Age]: if (TIME = INITIAL_TIME) then 0 else if ((growth - PREVIOUS(growth)) = 0) OR ((pop - PREVIOUS(pop)) = 0) then 0 else SAFEDIV(((PREVIOUS(pop[state, age·young]) * pop * 0.01) - PREVIOUS(growth)), ABS((growth - PREVIOUS(growth))), 0) * SIGN((pop - PREVIOUS(pop))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_mixed_occurrences.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_mixed_occurrences.txt new file mode 100644 index 000000000..f38db6b40 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_mixed_occurrences.txt @@ -0,0 +1,10 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→mixed[boston,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((mixed[region·boston,age·old] - PREVIOUS(mixed[region·boston,age·old])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·old])) - PREVIOUS(mixed[region·boston,age·old])), ABS((mixed[region·boston,age·old] - PREVIOUS(mixed[region·boston,age·old]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[boston,young]→mixed[boston,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((mixed[region·boston,age·young] - PREVIOUS(mixed[region·boston,age·young])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * PREVIOUS(pop[region·boston, age·young])) - PREVIOUS(mixed[region·boston,age·young])), ABS((mixed[region·boston,age·young] - PREVIOUS(mixed[region·boston,age·young]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→mixed[nyc,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((mixed[region·nyc,age·old] - PREVIOUS(mixed[region·nyc,age·old])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·nyc, age·old])) - PREVIOUS(mixed[region·nyc,age·old])), ABS((mixed[region·nyc,age·old] - PREVIOUS(mixed[region·nyc,age·old]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→mixed[nyc,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((mixed[region·nyc,age·young] - PREVIOUS(mixed[region·nyc,age·young])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * PREVIOUS(pop[region·nyc, age·young])) - PREVIOUS(mixed[region·nyc,age·young])), ABS((mixed[region·nyc,age·young] - PREVIOUS(mixed[region·nyc,age·young]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop→mixed dims=[Region,Age] +a2a[Region,Age]: if (TIME = INITIAL_TIME) then 0 else if ((mixed - PREVIOUS(mixed)) = 0) OR ((pop - PREVIOUS(pop)) = 0) then 0 else SAFEDIV(((PREVIOUS(pop[region, age·young]) * pop) - PREVIOUS(mixed)), ABS((mixed - PREVIOUS(mixed))), 0) * SIGN((pop - PREVIOUS(pop))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/per_element_other_deps.txt b/src/simlin-engine/src/db/ltm_char_golden/per_element_other_deps.txt new file mode 100644 index 000000000..ae8cd2df3 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/per_element_other_deps.txt @@ -0,0 +1,8 @@ +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston,age·old] - PREVIOUS(growth[region·boston,age·old])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * previous(w[age·old]) * previous(v[age·old, region·boston])) - PREVIOUS(growth[region·boston,age·old])), ABS((growth[region·boston,age·old] - PREVIOUS(growth[region·boston,age·old]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[boston,young]→growth[boston,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston,age·young] - PREVIOUS(growth[region·boston,age·young])) = 0) OR ((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young])) = 0) then 0 else SAFEDIV(((pop[boston, young] * previous(w[age·young]) * previous(v[age·young, region·boston])) - PREVIOUS(growth[region·boston,age·young])), ABS((growth[region·boston,age·young] - PREVIOUS(growth[region·boston,age·young]))), 0) * SIGN((pop[region·boston,age·young] - PREVIOUS(pop[region·boston,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc,old] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc,age·old] - PREVIOUS(growth[region·nyc,age·old])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * previous(w[age·old]) * previous(v[age·old, region·nyc])) - PREVIOUS(growth[region·nyc,age·old])), ABS((growth[region·nyc,age·old] - PREVIOUS(growth[region·nyc,age·old]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) +$⁚ltm⁚link_score⁚pop[nyc,young]→growth[nyc,young] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc,age·young] - PREVIOUS(growth[region·nyc,age·young])) = 0) OR ((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young])) = 0) then 0 else SAFEDIV(((pop[nyc, young] * previous(w[age·young]) * previous(v[age·young, region·nyc])) - PREVIOUS(growth[region·nyc,age·young])), ABS((growth[region·nyc,age·young] - PREVIOUS(growth[region·nyc,age·young]))), 0) * SIGN((pop[region·nyc,age·young] - PREVIOUS(pop[region·nyc,age·young]))) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs new file mode 100644 index 000000000..f99e713c3 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -0,0 +1,1120 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! Characterization pins for the LTM synthetic-equation generators that the +//! Track A "invert the composition" restructuring touches +//! (`generate_per_element_link_equation`, `generate_agg_to_scalar_target_equation`, +//! `generate_scalar_to_element_equation`, and the `Ast::Arrayed`-slot path in +//! `build_arrayed_link_score_equation`). +//! +//! These tests pin the EXACT generated `LtmSyntheticVar` equation text for a +//! battery of models that exercise every affected generator. They define +//! "behavior preserved": the restructuring (which moves row-pinning and +//! agg-substitution from PRE-transform rewrites to POST-transform lowerings) +//! must keep every byte of this text identical. A divergence here is either a +//! regression to fix or an adjudicated, documented, semantics-preserving change. +//! +//! The pins go through the production salsa entry point +//! (`model_ltm_variables`), so they exercise the full caller-to-generator data +//! flow (`db::ltm::link_scores`), not the generators in isolation. + +use super::*; +use crate::datamodel; +use crate::test_common::TestProject; + +/// Render one synthetic variable's equation as a stable, fully-structured +/// string: `Scalar`/`ApplyToAll` are their text; `Arrayed` lists each +/// `element => eqn` slot (element-sorted, as the generator emits) plus the +/// optional `default => eqn`. Dimensions are prefixed so a shape change is +/// visible in the pin. +fn render_equation(eq: &datamodel::Equation) -> String { + match eq { + datamodel::Equation::Scalar(s) => format!("scalar: {s}"), + datamodel::Equation::ApplyToAll(dims, s) => { + format!("a2a[{}]: {s}", dims.join(",")) + } + datamodel::Equation::Arrayed(dims, elements, default, apply_default) => { + let mut out = format!( + "arrayed[{}] (apply_default={apply_default}):", + dims.join(",") + ); + for (elem, eqn, _, _) in elements { + out.push_str(&format!("\n {elem} => {eqn}")); + } + if let Some(default_eqn) = default { + out.push_str(&format!("\n => {default_eqn}")); + } + out + } + } +} + +/// Deterministic dump of every synthetic variable whose name matches +/// `filter`, sorted by name, one `name` line followed by its rendered +/// equation. Used as the characterization surface: the whole string is +/// pinned byte-for-byte. +fn dump_synthetic_vars(project: datamodel::Project, discovery: bool, filter: &str) -> String { + use salsa::Setter; + let mut db = SimlinDb::default(); + let (source_project, model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + if discovery { + source_project.set_ltm_discovery_mode(&mut db).to(true); + } + let ltm = model_ltm_variables(&db, model, source_project); + let mut vars: Vec<&LtmSyntheticVar> = ltm + .vars + .iter() + .filter(|v| v.name.contains(filter)) + .collect(); + vars.sort_by(|a, b| a.name.cmp(&b.name)); + let mut out = String::new(); + for v in vars { + out.push_str(&v.name); + if !v.dimensions.is_empty() { + out.push_str(&format!(" dims=[{}]", v.dimensions.join(","))); + } + out.push('\n'); + out.push_str(&render_equation(&v.equation)); + out.push('\n'); + } + out +} + +/// Compare `actual` against the committed golden file `ltm_char_golden/{name}.txt`, +/// the byte-exact behavior pin. Set `UPDATE_LTM_GOLDEN=1` to (re)capture a +/// golden -- but only after the reviewer has adjudicated the change, per the +/// Track A stage-1 contract (an adjusted pin needs a documented +/// semantic-equivalence argument). A missing golden fails loudly. +#[track_caller] +fn assert_golden(name: &str, actual: &str) { + let path = format!( + "{}/src/db/ltm_char_golden/{name}.txt", + env!("CARGO_MANIFEST_DIR") + ); + if std::env::var("UPDATE_LTM_GOLDEN").is_ok() { + let dir = format!("{}/src/db/ltm_char_golden", env!("CARGO_MANIFEST_DIR")); + std::fs::create_dir_all(&dir).expect("create golden dir"); + std::fs::write(&path, actual).expect("write golden"); + return; + } + let expected = std::fs::read_to_string(&path).unwrap_or_else(|e| { + panic!("missing golden {path}: {e}; run once with UPDATE_LTM_GOLDEN=1 to capture") + }); + if actual != expected { + eprintln!("\n===== GOLDEN MISMATCH ({name}): actual below ====="); + eprintln!("{actual}"); + eprintln!("===== end actual (expected in {path}) =====\n"); + } + assert_eq!(actual, &expected, "golden mismatch for {name}"); +} + +// --------------------------------------------------------------------------- +// Model A: per-element (PerElement) link scores (GH #525 mixed iterated+literal) +// +// `growth[Region]` reads `pop[Region, young]` -- a mixed subscript: the Region +// axis is iterated (matches the A2A target dim), the Age axis is pinned to the +// literal `young`. This is `RefShape::PerElement`, routed to +// `emit_per_element_link_scores` -> `generate_per_element_link_equation`. +// --------------------------------------------------------------------------- + +fn per_element_model() -> datamodel::Project { + TestProject::new("per_element_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("pop[Region,Age]", "10") + .array_flow("growth[Region]", "pop[Region, young] * 0.1", None) + .array_stock("stock[Region]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_link_scores() { + let actual = dump_synthetic_vars(per_element_model(), true, "link_score\u{205A}pop"); + assert_golden("per_element_link_scores", &actual); +} + +// --------------------------------------------------------------------------- +// Model A2: per-element target over >=2 dims whose body also reads OTHER +// arrayed deps of (a) subset dims and (b) equal-arity reordered dims. +// +// `growth[Region,Age] = pop[Region, young] * w[Age] * v[Age, Region]`: +// * `pop[Region, young]` is the emitting `PerElement` site (Region iterated, +// Age pinned to `young`) -- the Iterated axis is a STRICT SUBSET of the +// target's `Region x Age` dims (the broadcast case). +// * `w[Age]` is an arrayed dep whose declared dims are a strict subset of the +// target's (an iterated-dim subscript over a single dim). +// * `v[Age, Region]` is an arrayed dep of EQUAL arity but REORDERED axes +// relative to the target's `Region, Age` iteration order. +// +// This is the other-dep arm of the inverted per-element wrap. It regression- +// guards that the ceteris-paribus wrap does NOT collapse these iterated other- +// dep subscripts to a bare `PREVIOUS(dep)` (which the post-transform element +// pin would then over-subscript with the FULL target tuple -- arity 2 over a +// 1-D `w`, a fragment that fails to compile and silently zeroes the score). +// Each dep must keep its subscript so the pin resolves each dimension-name +// index to this element's own coordinate: `w[age·y]` (arity 1) and +// `v[age·y, region·r]` (declared order). +// --------------------------------------------------------------------------- + +fn per_element_other_deps_model() -> datamodel::Project { + TestProject::new("per_element_other_deps_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("pop[Region,Age]", "10") + .array_aux("w[Age]", "0.5") + .array_aux("v[Age,Region]", "0.25") + .array_flow( + "growth[Region,Age]", + "pop[Region, young] * w[Age] * v[Age, Region]", + None, + ) + .array_stock("stock[Region,Age]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_other_deps() { + let actual = dump_synthetic_vars( + per_element_other_deps_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_other_deps", &actual); +} + +// --------------------------------------------------------------------------- +// Finding 1 materiality guard: a DYNAMIC feedback model whose per-element +// target body reads a SUBSET-dims arrayed dep. Unlike the constant +// characterization models above (whose text is pinned but whose scores are 0 +// because nothing changes), this one drives `growth` through a real feedback +// loop so the `pop[Region,young] -> growth[Region,Age]` link score is genuinely +// non-zero -- so the pre-fix over-subscription (`w[region·r, age·y]`, arity 2 +// over a 1-D `w`) is caught end-to-end: it made the fragment fail to compile +// (four `Assembly` `Warning`s) and silently zeroed every score series. This is +// the "pin gap hides a behavior change" guard: the text golden alone would not +// have distinguished the silent-0 runtime consequence. +// --------------------------------------------------------------------------- + +fn per_element_subset_dep_feedback_model() -> datamodel::Project { + TestProject::new("per_element_subset_dep_feedback") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("w[Age]", "0.5") + // The Region-young cohort's population drives growth of every element; + // `w[Age]` is a subset-dims dep whose pre-fix collapse broke the score. + .array_flow( + "growth[Region,Age]", + "pop[Region, young] * w[Age] * 0.01", + None, + ) + .array_stock("pop[Region,Age]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn per_element_subset_dep_scores_are_live_not_silent_zero() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = per_element_subset_dep_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + + // No LTM synthetic fragment may fail to compile: the pre-fix arity-2 + // `PREVIOUS(w[region·r, age·y])` over a 1-D `w` surfaced four of these. + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + assert!( + frag_failures.is_empty(), + "per-element link-score fragments must all compile (no silent \ + constant-0 stub); failed: {frag_failures:?}" + ); + + // ...and the compiled scores must actually be non-zero: a fragment that + // "compiles" to no bytecode reads a constant 0, so a series check is the + // ground-truth guard the diagnostic check backstops. + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed"); + let score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| k.as_str().contains("link_score\u{205A}pop")) + .map(|(_, v)| *v) + .collect(); + assert_eq!( + score_offsets.len(), + 4, + "expected four per-element pop->growth link scores in offsets" + ); + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + for off in score_offsets { + let has_nonzero = results.iter().any(|row| row[off] != 0.0); + assert!( + has_nonzero, + "per-element pop->growth link score at offset {off} is all-zero -- \ + the silent-degradation regression" + ); + } +} + +// --------------------------------------------------------------------------- +// Model A3: per-element target whose body carries an additional pop occurrence +// of DIFFERENT shape -- an all-iterated (`Bare`) live-source reference -- +// alongside the emitting `PerElement` site. +// +// `mixed[Region,Age] = pop[Region, young] * pop[Region, Age]`: +// * `pop[Region, young]` is the emitting `PerElement` site. +// * `pop[Region, Age]` is an all-iterated (`Bare`) occurrence -- a DIFFERENT +// shape, so the wrap freezes it (collapsing the iterated subscript to a +// bare `PREVIOUS(pop)` and same-element-pinning it via the LIVE-SOURCE +// path, which the other-dep fix leaves untouched) -- and it also seeds its +// own Bare A2A score. +// +// Pins that the frozen live-source occurrence lowers to the correct +// full-arity per-element read (`PREVIOUS(pop[region·r, age·a])`) regardless of +// the other-dep collapse decision: byte-for-byte stable coverage guarding +// against future drift in the live-source path. +// --------------------------------------------------------------------------- + +fn per_element_mixed_occurrences_model() -> datamodel::Project { + TestProject::new("per_element_mixed_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("pop[Region,Age]", "10") + .array_flow( + "mixed[Region,Age]", + "pop[Region, young] * pop[Region, Age]", + None, + ) + .array_stock("stock[Region,Age]", "0", &["mixed"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_mixed_occurrences() { + let actual = dump_synthetic_vars( + per_element_mixed_occurrences_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_mixed_occurrences", &actual); +} + +// --------------------------------------------------------------------------- +// Model A4 (finding 1): per-element target over positionally-MAPPED dims whose +// body carries an all-iterated occurrence of the live source alongside the +// emitting `PerElement` site. +// +// `growth[State,Age] = pop[State, young] * pop[State, Age] * 0.01`, with `State` +// positionally mapped to `Region` and `pop` declared over `Region x Age`: +// * `pop[State, young]` is the emitting `PerElement` site (State iterated and +// mapped to Region, Age pinned to `young`). +// * `pop[State, Age]` is an all-iterated (`Bare`-shaped) occurrence whose +// `State` axis is a MAPPED reference to `pop`'s declared `Region` axis. +// +// The wrap freezes the all-iterated occurrence at `PREVIOUS`. The row-pinning +// lowering must resolve its `State` index through the `State -> Region` +// correspondence, yielding `PREVIOUS(pop[region·, age·])`. The +// pre-fix inverted composition collapsed the iterated live-source subscript to +// a bare `PREVIOUS(pop)` BEFORE the lowering could see its indices, and the +// bare-`Var` pin -- which projects by the source's OWN dims -- found no row in +// the target-keyed projection, leaving an over-arity `PREVIOUS(pop)` that fails +// to compile and silently zeroes the score. The suppression of the live-source +// collapse under the `PerElement` live shape keeps the subscript alive so the +// mapped projection resolves. Byte-identical to HEAD. +// --------------------------------------------------------------------------- + +fn per_element_mapped_occurrence_model() -> datamodel::Project { + TestProject::new("per_element_mapped_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .named_dimension_with_mapping("State", &["west", "east"], "Region") + .array_aux("pop[Region,Age]", "10") + .array_flow( + "growth[State,Age]", + "pop[State, young] * pop[State, Age] * 0.01", + None, + ) + .array_stock("stock[State,Age]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_mapped_occurrence() { + let actual = dump_synthetic_vars( + per_element_mapped_occurrence_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_mapped_occurrence", &actual); +} + +// Finding 1 materiality guard: the mapped analogue of +// `per_element_subset_dep_scores_are_live_not_silent_zero`. A DYNAMIC model +// whose per-element target reads the live source through a positionally-MAPPED +// all-iterated occurrence: `pop` grows over time (a stock fed by a +// Region-dimensioned inflow -- no mapping in the flow->stock wiring, only in +// the `growth[State,*]` reads), so the `pop[State,young] -> growth[State,Age]` +// per-element link scores are genuinely non-zero. Pre-fix, the wrap collapsed +// the mapped `pop[State,Age]` occurrence to a bare `PREVIOUS(pop)` (arity 2 over +// the scalar fragment) that failed to compile -- four `Assembly` warnings and +// all-zero score series. The constant-`pop` text golden alone could not catch +// that runtime consequence. + +fn per_element_mapped_feedback_model() -> datamodel::Project { + TestProject::new("per_element_mapped_feedback") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .named_dimension_with_mapping("State", &["west", "east"], "Region") + .array_stock("pop[Region,Age]", "10", &["popinflow"], &[], None) + .array_flow("popinflow[Region,Age]", "1", None) + .array_flow( + "growth[State,Age]", + "pop[State, young] * pop[State, Age] * 0.01", + None, + ) + .array_stock("stock[State,Age]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn per_element_mapped_occurrence_scores_are_live_not_silent_zero() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = per_element_mapped_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + // Discovery mode scores every causal edge, so the `pop -> growth` + // per-element scores are emitted without needing a mapped flow->stock loop + // (which the dimension checker would reject). + source_project.set_ltm_discovery_mode(&mut db).to(true); + + // No LTM synthetic fragment may fail to compile: the pre-fix collapse + // produced arity-2 `PREVIOUS(pop)` fragments that surfaced four of these. + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + assert!( + frag_failures.is_empty(), + "per-element mapped-occurrence link-score fragments must all compile \ + (no silent constant-0 stub); failed: {frag_failures:?}" + ); + + // ...and the per-element scores must actually be non-zero. Filter on the + // `->growth[]` per-element targets so the Bare A2A `pop->growth` slots + // and the `popinflow->pop` edge are excluded. + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed"); + let score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| { + k.as_str().contains("link_score\u{205A}pop[") && k.as_str().contains("\u{2192}growth[") + }) + .map(|(_, v)| *v) + .collect(); + assert_eq!( + score_offsets.len(), + 4, + "expected four per-element mapped pop->growth link scores in offsets" + ); + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + for off in score_offsets { + let has_nonzero = results.iter().any(|row| row[off] != 0.0); + assert!( + has_nonzero, + "mapped per-element pop->growth link score at offset {off} is all-zero \ + -- the silent-degradation regression" + ); + } +} + +// --------------------------------------------------------------------------- +// Model A5 (Track A3 stage 1, finding 1): per-element target whose body carries +// an all-`Pinned` (literal) live-source occurrence whose element name belongs to +// MULTIPLE dimensions. +// +// `growth[Region,Age] = pop[Region, young] * pop[boston, old]`, with `boston` +// declared by BOTH `Region` and `Cities`: +// * `pop[Region, young]` is the emitting `PerElement` site. +// * `pop[boston, old]` is an all-`Pinned` (FixedIndex) occurrence. `boston` is +// ambiguous (Region + Cities), so the wrap's `qualify_element_index` (which +// only qualifies a name declared by exactly ONE dimension) declines it. +// +// The row-pinning lowering must still fully qualify the frozen occurrence from +// the SOURCE's own declared dims (`pop` is `Region x Age`), yielding +// `PREVIOUS(pop[region·boston, age·old])` -- NOT the half-qualified +// `PREVIOUS(pop[boston, age·old])` (where the wrap qualified `old -> age·old` +// but left the ambiguous `boston` bare, and the lowering's per-axis classifier +// could no longer classify the half-qualified subscript to re-qualify it). The +// fix suppresses the wrap's generic index qualification for the live source's +// own frozen subscript on the `PerElement` path, so the lowering owns +// qualification via `from_dims` (the single, ambiguity-free owner). Byte- +// identical to HEAD `f057ef38`. +// --------------------------------------------------------------------------- + +fn per_element_ambiguous_pin_model() -> datamodel::Project { + TestProject::new("per_element_ambiguous_pin_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Cities", &["boston", "la"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("pop[Region,Age]", "10") + .array_flow( + "growth[Region,Age]", + "pop[Region, young] * pop[boston, old]", + None, + ) + .array_stock("stock[Region,Age]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_ambiguous_pin() { + let actual = dump_synthetic_vars( + per_element_ambiguous_pin_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_ambiguous_pin", &actual); +} + +// Finding 1 materiality guard: a DYNAMIC feedback model whose per-element target +// body carries an all-`Pinned` live-source occurrence over an AMBIGUOUS element. +// `pop` grows over time so the `pop[Region,young] -> growth[Region,Age]` scores +// are genuinely non-zero. The pre-fix half-qualified `PREVIOUS(pop[boston, ...])` +// still happened to compile (positional axis resolution keeps bare `boston` +// resolvable), so unlike the collapse regressions this shape did NOT surface a +// compile failure -- which is exactly why the text-pin gap was material: a +// value-divergent change in this shape would have slipped through silently. This +// guard freezes the runtime equivalence: every per-element score is non-zero and +// the fragments all compile, in the same tree the text golden pins. + +fn per_element_ambiguous_pin_feedback_model() -> datamodel::Project { + TestProject::new("per_element_ambiguous_pin_feedback") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Cities", &["boston", "la"]) + .named_dimension("Age", &["young", "old"]) + .array_flow( + "growth[Region,Age]", + "pop[Region, young] * pop[boston, old] * 0.001", + None, + ) + .array_stock("pop[Region,Age]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn per_element_ambiguous_pin_scores_are_live_not_silent_zero() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = per_element_ambiguous_pin_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + assert!( + frag_failures.is_empty(), + "per-element ambiguous-pin link-score fragments must all compile; \ + failed: {frag_failures:?}" + ); + + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed"); + let score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| { + k.as_str().contains("link_score\u{205A}pop[") && k.as_str().contains("\u{2192}growth[") + }) + .map(|(_, v)| *v) + .collect(); + assert!( + !score_offsets.is_empty(), + "expected per-element pop[.,young]->growth link scores in offsets" + ); + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + for off in score_offsets { + let has_nonzero = results.iter().any(|row| row[off] != 0.0); + assert!( + has_nonzero, + "per-element ambiguous-pin link score at offset {off} is all-zero" + ); + } +} + +// --------------------------------------------------------------------------- +// Model A6 (Track A3 stage 1, finding 2, ADJUDICATED divergence): a live-source +// `PerElement` occurrence nested inside ANOTHER dep's subscript index. +// +// `growth[Region] = pop[Region, young] * other[pop[Region, young]]`: +// * the direct `pop[Region, young]` is the emitting `PerElement` site (held +// live -> bare row `pop[, young]`). +// * the nested `pop[Region, young]` inside `other[...]`'s index is a SECOND +// occurrence of the same site shape. The wrap freezes the enclosing +// `other[...]` at `PREVIOUS` (an other-dep), so the row-pinning lowering +// descends into it with `force_qualified` and takes its qualified arm. +// +// This pin captures the transform-first output `PREVIOUS(other[pop[region·, +// age·young]])` (the nested occurrence fully qualified from `pop`'s declared +// dims). It DIVERGES from HEAD `f057ef38`, which emitted the bare-row form +// `PREVIOUS(other[pop[, young]])`: HEAD's pre-transform rewrite pinned the +// nested occurrence BEFORE the wrap inserted the `PREVIOUS`, so it ran with +// `force_qualified == false` and produced the bare row. The divergence is +// semantics-preserving -- both spellings statically resolve to the identical +// `pop` element used as a dynamic index into `other`, and the +// `per_element_index_nested_scores_are_live_not_silent_zero` guard below proves +// the compiled score fragments are live (non-zero, finite, all compile). The +// transform-first lowering +// consistently qualifies EVERY frozen source occurrence from the source's own +// declared dims (the same rule that fixes finding 1's ambiguous all-`Pinned` +// occurrence); matching HEAD's incidental bare form here would require the +// lowering to distinguish a wrap-inserted `PREVIOUS` from an original-equation +// one, fragile machinery that de-simplifies the very transform this stage +// clarifies. Adjudicated as accepted per the stage-1 contract (see +// trackA3s1-report.md §3). +// --------------------------------------------------------------------------- + +fn per_element_index_nested_model() -> datamodel::Project { + TestProject::new("per_element_index_nested_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("pop[Region,Age]", "10") + .array_aux("other[Region]", "0.5") + .array_flow( + "growth[Region]", + "pop[Region, young] * other[pop[Region, young]]", + None, + ) + .array_stock("stock[Region]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_index_nested_occurrence() { + let actual = dump_synthetic_vars( + per_element_index_nested_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_index_nested_occurrence", &actual); +} + +// Finding 2 materiality guard: a DYNAMIC model whose per-element target reads +// the live source through an index-nested occurrence. `pop` grows over time, so +// the `pop -> growth` per-element scores (whose equations carry the qualified +// nested-index spelling this pin adjudicates) are genuinely non-zero. The index +// stays in range (small `pop`, wide `Slot`), so `other[pop[...]]` is finite and +// the score is not NaN-poisoned. This freezes the adjudicated spelling's runtime +// behavior: every per-element `pop -> growth` score is finite and non-zero and +// every LTM fragment compiles -- so a future value-divergent change to the +// index-nested lowering turns this red, closing the pin gap the review named. + +fn per_element_index_nested_feedback_model() -> datamodel::Project { + TestProject::new("per_element_index_nested_feedback") + .with_sim_time(0.0, 2.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .named_dimension("Slot", &["s0", "s1", "s2", "s3", "s4"]) + .array_aux("other[Slot]", "0.5") + .array_stock("pop[Region,Age]", "1", &["popinflow"], &[], None) + .array_flow("popinflow[Region,Age]", "0.5", None) + .array_flow( + "growth[Region]", + "pop[Region, young] * other[pop[Region, young]]", + None, + ) + .array_stock("stock[Region]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn per_element_index_nested_scores_are_live_not_silent_zero() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = per_element_index_nested_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + assert!( + frag_failures.is_empty(), + "per-element index-nested link-score fragments must all compile; \ + failed: {frag_failures:?}" + ); + + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed"); + let score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| { + k.as_str().contains("link_score\u{205A}pop[") && k.as_str().contains("\u{2192}growth[") + }) + .map(|(_, v)| *v) + .collect(); + assert!( + !score_offsets.is_empty(), + "expected per-element pop->growth link scores in offsets" + ); + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + for off in score_offsets { + let mut saw_nonzero = false; + for row in results.iter() { + let v = row[off]; + assert!( + v.is_finite(), + "index-nested pop->growth score at offset {off} is non-finite \ + ({v}) -- the qualified nested-index spelling must stay in range" + ); + saw_nonzero |= v != 0.0; + } + assert!( + saw_nonzero, + "index-nested pop->growth link score at offset {off} is all-zero" + ); + } +} + +// --------------------------------------------------------------------------- +// Model A7 (Track A3 stage 1, finding 1): per-element target whose body carries +// a frozen live-source occurrence with a DYNAMIC (non-element, non-dim-name) +// index. +// +// `growth[Region] = pop[Region, young] * pop[Region, idx] * 0.001`, `idx = 1 + +// (TIME MOD 2)`: +// * `pop[Region, young]` is the emitting `PerElement` site. +// * `pop[Region, idx]` is a frozen occurrence whose Age axis is the DYNAMIC +// index `idx` -- a genuine ceteris-paribus dependency, not an element +// selector. +// +// The frozen occurrence must lower to `PREVIOUS(pop[region·, PREVIOUS(idx)])`: +// the iterated `Region` axis pinned+qualified by the lowering, and the dynamic +// `idx` WRAPPED in `PREVIOUS` by the wrap's index pass (its ceteris-paribus lag). +// The finding-1 index-qualification suppression must be scoped to element +// qualification ONLY -- a blanket "keep the indices pristine" would drop the +// `PREVIOUS(idx)` lag, emitting `PREVIOUS(pop[region·, idx])` and changing the +// compiled score series (HEAD: a NaN at the first live step; the un-lagged form: +// finite). Byte-identical to HEAD `f057ef38`. +// --------------------------------------------------------------------------- + +fn per_element_dynamic_index_model() -> datamodel::Project { + TestProject::new("dyn_index_char") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .scalar_aux("idx", "1 + (TIME MOD 2)") + .array_flow( + "growth[Region]", + "pop[Region, young] * pop[Region, idx] * 0.001", + None, + ) + .array_stock("pop[Region,Age]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_per_element_dynamic_index() { + let actual = dump_synthetic_vars( + per_element_dynamic_index_model(), + true, + "link_score\u{205A}pop", + ); + assert_golden("per_element_dynamic_index", &actual); +} + +// Finding 1 materiality guard: the DYNAMIC-index sibling of the ambiguous-pin +// guard. The frozen `pop[Region, idx]` occurrence lowers to a `PREVIOUS(idx)` +// lag; the buggy blanket-skip would instead leave `idx` un-lagged, a +// value-divergent change the constant-`pop` text golden alone would still catch +// (the golden diverges) but whose RUNTIME consequence this guard freezes. HEAD's +// series carries a NaN at the first live step (a pre-existing HEAD behavior: the +// synthesized `PREVIOUS(idx)` capture-helper aux reads an uninitialized value at +// the initial step; tracked separately as a latent bug). Reproducing that NaN is +// the byte-parity contract; the un-lagged regression makes the same step FINITE, +// so `results[1]` being NaN is the exact discriminator. This guard therefore +// pins (a) every fragment compiles (the `PREVIOUS(idx)` helper is well-formed), +// (b) the first-live-step score is NaN (the dynamic-index lag is present), and +// (c) a later step is finite and non-zero (the score is materially live). +fn per_element_dynamic_index_feedback_model() -> datamodel::Project { + // Same shape as `per_element_dynamic_index_model` (pop is already a + // growth-fed stock there), reused directly so the guard and the text pin + // exercise the identical model. + per_element_dynamic_index_model() +} + +#[test] +fn per_element_dynamic_index_scores_preserve_head_lag() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = per_element_dynamic_index_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + assert!( + frag_failures.is_empty(), + "dynamic-index per-element link-score fragments must all compile; \ + failed: {frag_failures:?}" + ); + + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed"); + // The real per-element scores start with the single `$⁚ltm⁚link_score⁚` + // prefix; the synthesized `PREVIOUS`-capture helper auxes carry a `$⁚$⁚` + // double prefix, so `starts_with` excludes them. + let score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| { + k.as_str() + .starts_with("$\u{205A}ltm\u{205A}link_score\u{205A}pop[") + && k.as_str().contains("\u{2192}growth[") + }) + .map(|(_, v)| *v) + .collect(); + assert_eq!( + score_offsets.len(), + 2, + "expected two per-element dynamic-index pop->growth link scores" + ); + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + let rows: Vec<_> = results.iter().collect(); + for off in score_offsets { + // Step 1 (the first live step) reproduces HEAD's NaN, the signature of + // the `PREVIOUS(idx)` lag reading its uninitialized capture helper. An + // un-lagged regression (`idx` not wrapped) makes this step FINITE. + assert!( + rows[1][off].is_nan(), + "dynamic-index pop->growth score at offset {off} step 1 must be NaN \ + (the preserved PREVIOUS(idx) dynamic-index lag); a finite value means \ + the lag was dropped" + ); + // A later step is finite and non-zero: the score is materially live. + let has_finite_nonzero = rows + .iter() + .skip(2) + .any(|row| row[off].is_finite() && row[off] != 0.0); + assert!( + has_finite_nonzero, + "dynamic-index pop->growth score at offset {off} has no finite non-zero \ + step -- the score is not materially live" + ); + } +} + +// --------------------------------------------------------------------------- +// Model B: agg -> scalar target (synthetic agg, `generate_agg_to_scalar_target_equation`) +// +// `total = SUM(pop[*]) * 2` -- the reducer is a SUBexpression (not the whole +// RHS), so it is hoisted into a synthetic `$⁚ltm⁚agg⁚0` and the `agg -> total` +// half runs through the scalar-target generator on the agg-substituted text. +// --------------------------------------------------------------------------- + +fn agg_to_scalar_model() -> datamodel::Project { + TestProject::new("agg_to_scalar_char") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "10") + .scalar_aux("total", "SUM(pop[*]) * 2") + .stock("acc", "0", &["totalflow"], &[], None) + .aux("totalflow", "total", None) + .build_datamodel() +} + +#[test] +fn char_agg_to_scalar_target() { + let actual = dump_synthetic_vars(agg_to_scalar_model(), true, "\u{205A}agg\u{205A}0\u{2192}"); + assert_golden("agg_to_scalar_target", &actual); +} + +// --------------------------------------------------------------------------- +// Model B2 (Track A3 stage 1, finding 2): a hoisted reducer NESTED inside a +// DECLINED (non-hoisted) outer reducer. +// +// `growth[Region] = SUM(matrix[Region,*] * other[nyc,*] * SUM(pop[*])) * 0.001`: +// * the inner `SUM(pop[*])` is a whole-extent reduce -> hoisted into +// `$⁚ltm⁚agg⁚0`. +// * the outer `SUM(...)` is DECLINED by the I1 acceptance (its co-source slices +// differ: `matrix[Region,*]` is `[Iterated, Reduced]` while `other[nyc,*]` is +// `[Pinned, Reduced]`), so it stays inline. +// +// The `agg⁚0 -> growth[e]` half wraps the target's own equation with the inner +// reducer held LIVE by `live_reducer_text`. The wrap must NOT freeze the whole +// declined outer reducer: doing so (the GH #517 arm) would drop the live agg +// entirely, yielding `PREVIOUS(SUM(matrix[..] * other[..] * SUM(pop[*])))` -- a +// clean-compiling structural zero (the frozen partial equals the frozen anchor). +// HEAD instead recurses into the outer reducer, holds the agg live, and freezes +// the co-source array slices: `sum(previous(matrix[region·, *]) * +// previous(other[region·nyc, *]) * "$⁚ltm⁚agg⁚0") * 0.001` -- which fails to +// compile (no LoadPrev-of-array-view path) and surfaces as a LOUD warned zero. +// The finding-2 gate teaches the GH #517 freeze about `live_reducer_text` so the +// enclosing reducer recurses, restoring HEAD's text (and its loud degradation -- +// a loud failure is strictly better than a silent structural zero). Byte- +// identical to HEAD `f057ef38`. +// --------------------------------------------------------------------------- + +fn agg_nested_reducer_model() -> datamodel::Project { + TestProject::new("nested_reducer_char") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "10") + .array_aux("matrix[Region,Region]", "1") + .array_aux("other[Region,Region]", "1") + .array_flow( + "growth[Region]", + "SUM(matrix[Region,*] * other[nyc,*] * SUM(pop[*])) * 0.001", + None, + ) + .array_stock("stock[Region]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_agg_nested_reducer() { + let actual = dump_synthetic_vars(agg_nested_reducer_model(), true, "link_score"); + assert_golden("agg_nested_reducer", &actual); +} + +// Finding 2 materiality guard: unlike every other guard in this file (which +// asserts fragments MUST compile), the byte-parity contract here is to REPRODUCE +// HEAD's LOUD failure. HEAD emits the live-agg-inside-a-frozen-array-slice +// partial that cannot compile (six `Assembly` "failed to compile" warnings: the +// two `agg⁚0->growth[e]` scores plus their four `PREVIOUS`-capture helper auxes), +// producing a warned zero. The pre-fix transform-first freeze produced a SILENT +// clean-compiling zero (zero warnings) -- the worst failure class. This guard +// pins that the loud warnings ARE present: a regression that silently zeroes the +// score would drop them. `pop` is a stock so the edge is causally live. +fn agg_nested_reducer_feedback_model() -> datamodel::Project { + TestProject::new("nested_reducer_feedback") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("matrix[Region,Region]", "1") + .array_aux("other[Region,Region]", "1") + .array_flow( + "growth[Region]", + "SUM(matrix[Region,*] * other[nyc,*] * SUM(pop[*])) * 0.001", + None, + ) + .array_stock("pop[Region]", "10", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn agg_nested_reducer_preserves_loud_failure_not_silent_zero() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = agg_nested_reducer_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + + let diags = collect_model_diagnostics(&db, source_model, source_project); + let frag_failures: Vec = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + // The loud degradation must be present: the two agg⁚0->growth[e] scores must + // each fail to compile (the live-agg-inside-a-frozen-array-slice partial), + // matching HEAD. A silent-zero regression would report NO such failure. + let agg_growth_failures: Vec<&String> = frag_failures + .iter() + .filter(|v| v.contains("agg\u{205A}0\u{2192}growth")) + .collect(); + assert!( + agg_growth_failures.len() >= 2, + "the nested-reducer agg->growth partial must LOUDLY fail to compile \ + (HEAD's warned zero), not silently zero; failed fragments: {frag_failures:?}" + ); +} + +// --------------------------------------------------------------------------- +// Model C: agg -> arrayed target (`generate_scalar_to_element_equation`, scalar agg) +// +// `outflow[Region] = SUM(pop[*]) * frac[Region]` -- whole-extent SUM hoisted +// into a scalar synthetic agg; the `agg -> outflow[e]` half is one scalar var +// per target element via the scalar-to-element generator. +// --------------------------------------------------------------------------- + +fn agg_to_arrayed_model() -> datamodel::Project { + TestProject::new("agg_to_arrayed_char") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "10") + .array_aux("frac[Region]", "0.5") + .array_flow("outflow[Region]", "SUM(pop[*]) * frac[Region]", None) + .array_stock("stock[Region]", "0", &["outflow"], &[], None) + .build_datamodel() +} + +#[test] +fn char_agg_to_arrayed_target() { + let actual = dump_synthetic_vars(agg_to_arrayed_model(), true, "\u{205A}agg\u{205A}0\u{2192}"); + assert_golden("agg_to_arrayed_target", &actual); +} + +// --------------------------------------------------------------------------- +// Model D: arrayed agg -> arrayed target with `source_pins` (GH #528) +// +// `outflow[Region] = SUM(matrix[Region,*]) * 0.1` -- a SLICED reducer hoisted +// into an ARRAYED agg over Region; the `agg -> outflow[e]` half pins the agg +// to the target-element projection (`source_pins`), exercising the arrayed-agg +// branch of `generate_scalar_to_element_equation`. +// --------------------------------------------------------------------------- + +fn arrayed_agg_model() -> datamodel::Project { + TestProject::new("arrayed_agg_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Dim2", &["a", "b"]) + .array_aux("matrix[Region,Dim2]", "10") + .array_flow("outflow[Region]", "SUM(matrix[Region,*]) * 0.1", None) + .array_stock("stock[Region]", "0", &["outflow"], &[], None) + .build_datamodel() +} + +#[test] +fn char_arrayed_agg_to_target() { + let actual = dump_synthetic_vars(arrayed_agg_model(), true, "link_score"); + assert_golden("arrayed_agg_to_target", &actual); +} + +// --------------------------------------------------------------------------- +// Model E: Ast::Arrayed (per-element-equation) target (Category B, +// `build_arrayed_link_score_equation`) +// +// `mp` has per-element equations referencing `pop` by fixed element subscripts; +// each slot's link score is the guard form built on THAT slot's own text. +// --------------------------------------------------------------------------- + +fn arrayed_target_model() -> datamodel::Project { + let mut p = TestProject::new("arrayed_target_char") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "10"); + // A per-element-equation (Ast::Arrayed) target: each element has its own eqn. + p = p.array_with_default_and_overrides( + "mp[Region]", + "0", + vec![ + ("nyc", "(pop[nyc] - pop[boston]) * 0.01"), + ("boston", "(pop[boston] - pop[nyc]) * 0.01"), + ], + ); + p.array_stock("stock[Region]", "0", &["mpflow"], &[], None) + .array_flow("mpflow[Region]", "mp", None) + .build_datamodel() +} + +#[test] +fn char_arrayed_target_slot_scores() { + let actual = dump_synthetic_vars(arrayed_target_model(), true, "link_score\u{205A}pop"); + assert_golden("arrayed_target_slot_scores", &actual); +} diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 12c2f9735..1019d938e 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -31,6 +31,24 @@ pub(crate) use with_lookup::{ WithLookupSlotRefs, WithLookupWrap, with_lookup_reducer_owner_wrap, with_lookup_table_ref, }; +/// The post-transform lowering machinery (Track A stage 1), in its own file +/// only to keep this one under the project line-count lint. These lowerings run +/// AFTER the ceteris-paribus wrap on the target's own equation, so the wrap +/// stays occurrence-addressable for stage 2/3; see the module rustdoc. The +/// parent-only helpers are imported for the `generate_*` callers below; the +/// externally-referenced row derivation and the test-only text wrapper are +/// re-exported so callers keep naming them `crate::ltm_augment::*`. +#[path = "ltm_augment_post_transform.rs"] +mod post_transform; + +pub(crate) use post_transform::per_element_row_for_target; +#[cfg(test)] +pub(crate) use post_transform::substitute_reducers_in_equation; +use post_transform::{ + PerElementRefCtx, qualify_axis_element, rewrite_per_element_source_refs, + substitute_reducers_in_expr0, +}; + /// Context for recognizing GH #511 iterated-dimension source references in /// the partial-equation builder: the live source's declared dimension names /// (canonical, in declaration order; same length as `source_dim_elements`), @@ -566,6 +584,51 @@ fn expr0_contains_live_match( } } +/// Whether any subexpression of `expr` prints exactly as `reducer_text` -- the +/// [`WrapCtx::live_reducer_text`] containment test (Track A stage 1, finding 2). +/// +/// The GH #517 arm of [`wrap_non_matching_in_previous`] freezes a whole array +/// reducer that carries no live reference. But when the hoisted reducer held +/// LIVE (matched by text at the top of the wrap) is itself NESTED inside a +/// DECLINED (non-hoisted) outer reducer -- `SUM(matrix[D,*] * SUM(pop[*]))`, +/// where the inner `SUM(pop[*])` became the agg -- the whole outer reducer would +/// freeze before the recursion ever reaches the inner one, silently converting +/// HEAD's live-agg-inside-a-frozen-slice partial (which fails to compile: a loud +/// warned zero) into a structurally-always-zero frozen partial. This predicate +/// lets that enclosing reducer recurse instead of freeze, so the inner +/// held-live reducer is reached (and later substituted to its agg name), +/// preserving HEAD's text and diagnostic surface. Only an `App` can be the +/// hoisted reducer, so the `print_eqn` comparison is confined to `App` nodes. +fn expr0_contains_reducer_text(expr: &Expr0, reducer_text: &str) -> bool { + match expr { + Expr0::Const(..) | Expr0::Var(..) => false, + Expr0::Subscript(_, indices, _) => indices.iter().any(|idx| match idx { + IndexExpr0::Expr(e) => expr0_contains_reducer_text(e, reducer_text), + IndexExpr0::Range(l, r, _) => { + expr0_contains_reducer_text(l, reducer_text) + || expr0_contains_reducer_text(r, reducer_text) + } + _ => false, + }), + Expr0::App(UntypedBuiltinFn(_, args), _) => { + print_eqn(expr) == reducer_text + || args + .iter() + .any(|a| expr0_contains_reducer_text(a, reducer_text)) + } + Expr0::Op1(_, inner, _) => expr0_contains_reducer_text(inner, reducer_text), + Expr0::Op2(_, l, r, _) => { + expr0_contains_reducer_text(l, reducer_text) + || expr0_contains_reducer_text(r, reducer_text) + } + Expr0::If(c, t, e, _) => { + expr0_contains_reducer_text(c, reducer_text) + || expr0_contains_reducer_text(t, reducer_text) + || expr0_contains_reducer_text(e, reducer_text) + } + } +} + /// Out-channels of [`wrap_non_matching_in_previous`], threaded through the /// recursion as one mutable sink. #[derive(Default)] @@ -586,6 +649,40 @@ struct WrapOutcome { other_dep_mismatch: bool, } +/// The immutable parameters of the ceteris-paribus wrap +/// ([`wrap_non_matching_in_previous`] / [`wrap_index_non_matching_in_previous`]), +/// bundled so the deep recursion threads one `&WrapCtx` instead of a long +/// positional argument list. Every field is a `Copy` reference, so the body +/// destructures it back into the historically-named locals with no clones. +struct WrapCtx<'a> { + /// The source variable whose live shape is held out of `PREVIOUS` + /// wrapping; all its other occurrences (and every other-dep reference) + /// are wrapped. + live_source: &'a Ident, + /// Which access shape of `live_source` stays live; other shapes wrap. + live_shape: &'a RefShape, + /// Track A stage 1 (opt-in): a hoisted reducer subexpression to hold LIVE + /// verbatim, matched by its canonical `print_eqn` text. When set, an `App` + /// whose printed form equals this is the live thing (recorded as + /// `live_ref`, left verbatim, never recursed into), and every OTHER + /// recognized reducer freezes whole (the GH #517 path). This lets the + /// caller run the wrap on the target's OWN equation and substitute all + /// reducers to their agg names AFTER the wrap -- the held-live one to a + /// bare agg name, the frozen co-reducers to `PREVIOUS(agg)` -- rather than + /// wrapping already-agg-substituted text (the inverted composition). `None` + /// for the ordinary ident-live callers, who are byte-for-byte unaffected. + live_reducer_text: Option<&'a str>, + /// Canonical idents of the non-`live_source` deps that must be wrapped; + /// names outside this set (and not `live_source`) are left alone. + other_deps: &'a HashSet>, + /// Per-source-axis element-name lists, in source-declared order. + source_dim_elements: &'a [Vec], + /// GH #511 iterated-dimension context (`None` for a scalar live source). + iter_ctx: Option<&'a IteratedDimCtx<'a>>, + /// Project dims context for [`qualify_element_index`] (GH #587). + dims_ctx: Option<&'a crate::dimensions::DimensionsContext>, +} + /// Walk an `Expr0` tree and wrap variable references in `PREVIOUS()` except /// those whose access shape matches the live shape for the given source, /// recording into `out.live_ref` the *first* `live_source` occurrence left @@ -631,17 +728,32 @@ struct WrapOutcome { /// they were causal references (GH #587). `None` (test-only callers, or /// paths without project dims in scope) disables qualification, keeping the /// conservative wrapping behavior. -#[allow(clippy::too_many_arguments)] -fn wrap_non_matching_in_previous( - expr: Expr0, - live_source: &Ident, - live_shape: &RefShape, - other_deps: &HashSet>, - source_dim_elements: &[Vec], - iter_ctx: Option<&IteratedDimCtx<'_>>, - dims_ctx: Option<&crate::dimensions::DimensionsContext>, - out: &mut WrapOutcome, -) -> Expr0 { +fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapOutcome) -> Expr0 { + // `dims_ctx` is consumed only by `wrap_index_non_matching_in_previous` (via + // `ctx`), so it is intentionally not bound here. + let &WrapCtx { + live_source, + live_shape, + live_reducer_text, + other_deps, + source_dim_elements, + iter_ctx, + .. + } = ctx; + // Track A stage 1: hold a designated hoisted reducer subexpression LIVE + // verbatim (matched by canonical text). Only an `App` can be a reducer, and + // the post-transform substitution renames it to its agg name in place. The + // check is skipped for the ordinary ident-live callers (`None`), so their + // output is byte-identical. See [`WrapCtx::live_reducer_text`]. + if let Some(reducer_text) = live_reducer_text + && matches!(expr, Expr0::App(..)) + && print_eqn(&expr) == reducer_text + { + if out.live_ref.is_none() { + out.live_ref = Some(expr.clone()); + } + return expr; + } match expr { Expr0::Const(..) => expr, Expr0::Var(ref ident, loc) => { @@ -681,31 +793,63 @@ fn wrap_non_matching_in_previous( // alternative -- `PREVIOUS(Subscript(...))` -- trips the // codegen assertion. if &canonical == live_source { - if is_live_source_iterated_dim_subscript(&indices, source_dim_elements, iter_ctx) { - return wrap_non_matching_in_previous( - Expr0::Var(ident, loc), - live_source, - live_shape, - other_deps, + // The live-source iterated-dim collapse (`pop[Region,Age]` -> + // bare `Var(pop)`) is only sound when the emitted partial keeps + // the source reference bare-and-iterated -- the A2A / scalar + // partial paths, where a `Bare` live shape then reads the + // current element under the target's own A2A expansion. The + // `PerElement` live shape is the inverted per-element path + // (`generate_per_element_link_equation`): it emits a SCALAR + // equation per (row, target element), so any OTHER live-source + // occurrence in the body (an all-iterated `pop[Region,Age]` + // alongside the emitting `pop[Region,young]` site) is frozen at + // `PREVIOUS` and then pinned to its own concrete row AFTER the + // wrap (`rewrite_per_element_source_refs`, keyed on the + // subscript's per-axis classification). Collapsing to bare here + // discards the subscript that pin needs: the bare-`Var` pin + // projects by the source's OWN dims, so for a positionally- + // MAPPED occurrence (`pop[State,Age]`, State->Region) it finds + // no row in the target-keyed projection and leaves an over-arity + // `PREVIOUS(pop)` that fails to compile and silently zeroes the + // score. Skip the collapse so the subscript survives; the frozen + // occurrence keeps `pop[State,Age]` and the post-transform pin + // resolves each iterated axis -- mapped axes through the + // `State->Region` correspondence -- to this element's own + // coordinate. (For the unmapped case the pin resolves the same + // row either way, so the output is byte-identical.) + if !matches!(live_shape, RefShape::PerElement { .. }) + && is_live_source_iterated_dim_subscript( + &indices, source_dim_elements, iter_ctx, - dims_ctx, - out, - ); + ) + { + return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out); } - } else if other_deps.contains(&canonical) { + } else if other_deps.contains(&canonical) + && !matches!(live_shape, RefShape::PerElement { .. }) + { + // The other-dep iterated-subscript collapse (`w[Age]` -> bare + // `PREVIOUS(w)`) is only sound when the emitted partial keeps + // the dep bare-and-iterated -- the A2A / scalar partial paths, + // whose result is an `Equation::ApplyToAll` that reads the + // current element. The `PerElement` live shape is the inverted + // per-element path (`generate_per_element_link_equation`): it + // emits a SCALAR equation per (row, target element) and pins + // every arrayed dep to that element afterward + // (`subscript_idents_at_element`). Collapsing to bare there + // would let the full-target-tuple pin over-subscript a + // subset-dims dep (`w[region·boston, age·old]` -- arity 2 over + // a 1-D `w`), producing a fragment that fails to compile and + // silently zeroes the score. Skip the collapse so the dep + // keeps its subscript (`PREVIOUS(w[Age])`) and the pin resolves + // each dimension-name index to this element's own coordinate + // (`PREVIOUS(w[age·old])`). The GH #526 `Mismatch` doom cannot + // fire on this path anyway (its `dep_dims` are `None`), so + // skipping the whole verdict preserves behavior. match classify_other_dep_iterated_dim_subscript(&canonical, &indices, iter_ctx) { OtherDepIteratedVerdict::Collapse => { - return wrap_non_matching_in_previous( - Expr0::Var(ident, loc), - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - ); + return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out); } OtherDepIteratedVerdict::Mismatch => { // GH #526: collapsing would freeze the WRONG element. @@ -757,16 +901,7 @@ fn wrap_non_matching_in_previous( if is_literal_element_index(&idx, i, source_dim_elements) { idx } else { - wrap_index_non_matching_in_previous( - idx, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - ) + wrap_index_non_matching_in_previous(idx, ctx, out, false) } }) .collect(); @@ -780,19 +915,44 @@ fn wrap_non_matching_in_previous( // user-variable references get wrapped, then build the new // subscript. If the outer ident is itself a dep, wrap the // whole thing. + // + // EXCEPTION (Track A stage 1, `PerElement` path): for a FROZEN + // occurrence of the LIVE SOURCE ITSELF, suppress only the wrap's + // generic element-index QUALIFICATION -- NOT the whole index pass. + // The post-transform row-pinning lowering + // (`rewrite_per_element_source_refs`) re-derives and qualifies every + // source-axis index from the source's KNOWN declared dims (one + // consistent, ambiguity-free owner), so letting the wrap's + // `qualify_element_index` touch a literal element first would + // HALF-qualify an AMBIGUOUS one (a name declared by several dims, + // like C-LEARN's region elements: in `pop[boston, old]`, `old` -> + // `age·old` but the ambiguous `boston` stays bare because + // `dimension_uniquely_containing_element` declines it). The + // lowering's per-axis classifier then cannot classify the resulting + // half-qualified subscript (a `dim·elem` index no longer matches the + // bare source-element list), so the ambiguous index is never + // re-qualified -- diverging from HEAD's `region·boston` (whose + // pre-wrap rewrite qualified via the source axis it knew). Leaving + // literal indices bare keeps the subscript classifiable so the + // lowering qualifies all of them from `from_dims`. + // + // A genuinely-DYNAMIC index (a non-element, non-dim-name expression + // like `idx` in `pop[Region, idx]`) is still recursed and wrapped: + // it is real ceteris-paribus content, and HEAD's rewrite-then-wrap + // composition wrapped it too (its pre-transform rewrite pinned only + // the iterated axis, leaving `idx` for the wrap's index pass to lag). + // Suppressing the ENTIRE index pass -- as a blanket "keep the indices + // pristine" would -- silently drops that `PREVIOUS(idx)` lag, changing + // both the emitted text and the compiled score series. So the skip is + // scoped to qualification only; the lowering, which leaves an + // unresolvable dynamic index untouched, then preserves the + // `PREVIOUS(idx)` the wrap produced. + let skip_index_qualification = + &canonical == live_source && matches!(live_shape, RefShape::PerElement { .. }); let indices: Vec = indices .into_iter() .map(|idx| { - wrap_index_non_matching_in_previous( - idx, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - ) + wrap_index_non_matching_in_previous(idx, ctx, out, skip_index_qualification) }) .collect(); let subscript = Expr0::Subscript(ident, indices, loc); @@ -835,18 +995,7 @@ fn wrap_non_matching_in_previous( let mut args_iter = args.into_iter(); let table_arg = args_iter.next().expect("checked non-empty"); let mut new_args = vec![table_arg]; - new_args.extend(args_iter.map(|a| { - wrap_non_matching_in_previous( - a, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - ) - })); + new_args.extend(args_iter.map(|a| wrap_non_matching_in_previous(a, ctx, out))); return Expr0::App(UntypedBuiltinFn(name, new_args), loc); } // GH #517: an array-reducer subexpression (`SUM(pop[*])`, @@ -862,8 +1011,19 @@ fn wrap_non_matching_in_previous( // If the live reference *is* inside this reducer (the now // test-only `RefShape::Wildcard` path where `SUM(pop[*])` is the // live thing), recurse normally so the live `pop[*]` stays - // unwrapped. + // unwrapped. Likewise (Track A stage 1, finding 2) if the reducer + // held LIVE by `live_reducer_text` is NESTED inside this one -- a + // hoisted `SUM(pop[*])` inside a DECLINED outer `SUM(matrix[D,*] * + // SUM(pop[*]))` -- recurse so the inner reducer is reached and held + // live; freezing the outer whole would drop the live reference + // entirely (a structural zero, vs HEAD's live-agg-inside-a-frozen- + // slice partial). The top-of-function guard has already declined to + // hold THIS reducer live (its own text does not equal + // `live_reducer_text`), so this only affects an enclosing reducer. + let holds_live_reducer = live_reducer_text + .is_some_and(|text| args.iter().any(|a| expr0_contains_reducer_text(a, text))); if is_array_reducer_name(&name, args.len()) + && !holds_live_reducer && !args.iter().any(|a| { expr0_contains_live_match( a, @@ -879,90 +1039,25 @@ fn wrap_non_matching_in_previous( } let args = args .into_iter() - .map(|a| { - wrap_non_matching_in_previous( - a, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - ) - }) + .map(|a| wrap_non_matching_in_previous(a, ctx, out)) .collect(); Expr0::App(UntypedBuiltinFn(name, args), loc) } Expr0::Op1(op, inner, loc) => Expr0::Op1( op, - Box::new(wrap_non_matching_in_previous( - *inner, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), + Box::new(wrap_non_matching_in_previous(*inner, ctx, out)), loc, ), Expr0::Op2(op, lhs, rhs, loc) => Expr0::Op2( op, - Box::new(wrap_non_matching_in_previous( - *lhs, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), - Box::new(wrap_non_matching_in_previous( - *rhs, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), + Box::new(wrap_non_matching_in_previous(*lhs, ctx, out)), + Box::new(wrap_non_matching_in_previous(*rhs, ctx, out)), loc, ), Expr0::If(cond, then_expr, else_expr, loc) => Expr0::If( - Box::new(wrap_non_matching_in_previous( - *cond, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), - Box::new(wrap_non_matching_in_previous( - *then_expr, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), - Box::new(wrap_non_matching_in_previous( - *else_expr, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - out, - )), + Box::new(wrap_non_matching_in_previous(*cond, ctx, out)), + Box::new(wrap_non_matching_in_previous(*then_expr, ctx, out)), + Box::new(wrap_non_matching_in_previous(*else_expr, ctx, out)), loc, ), } @@ -1012,22 +1107,32 @@ fn qualify_element_index( ))) } -#[allow(clippy::too_many_arguments)] fn wrap_index_non_matching_in_previous( index: IndexExpr0, - live_source: &Ident, - live_shape: &RefShape, - other_deps: &HashSet>, - source_dim_elements: &[Vec], - iter_ctx: Option<&IteratedDimCtx<'_>>, - dims_ctx: Option<&crate::dimensions::DimensionsContext>, + ctx: &WrapCtx<'_>, out: &mut WrapOutcome, + skip_element_qualification: bool, ) -> IndexExpr0 { + // Only `dims_ctx` (element/dimension recognition) and `iter_ctx` (the + // iterated-dim-name guard) are read directly here; the full wrap context + // rides `ctx` into the recursive `wrap_non_matching_in_previous` calls. + let &WrapCtx { + dims_ctx, iter_ctx, .. + } = ctx; // An index that unambiguously names a dimension element is an element // selector, never a causal reference: qualify it and leave it unwrapped // (GH #587). This must be checked BEFORE the recursive wrap below, which // would otherwise treat the element name as a dep reference. - if let Some(qualified) = qualify_element_index(&index, dims_ctx) { + // + // `skip_element_qualification` is set on the `PerElement` frozen-live-source + // path (Track A stage 1, finding 1): there the row-pinning lowering owns + // source-index qualification from `from_dims`, so a literal element index is + // left bare here (the element-verbatim guards below still keep it unwrapped) + // and the lowering qualifies it consistently. The recursive wrap still runs + // for a genuinely-dynamic index below, so a frozen `pop[Region, idx]` keeps + // its `PREVIOUS(idx)` lag. + if !skip_element_qualification && let Some(qualified) = qualify_element_index(&index, dims_ctx) + { return qualified; } // An index that names a dimension element which *cannot* be qualified @@ -1083,37 +1188,12 @@ fn wrap_index_non_matching_in_previous( // collapse dooms the changed-first partial just the same. let mut idx_out = WrapOutcome::default(); let result = match index { - IndexExpr0::Expr(e) => IndexExpr0::Expr(wrap_non_matching_in_previous( - e, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - &mut idx_out, - )), + IndexExpr0::Expr(e) => { + IndexExpr0::Expr(wrap_non_matching_in_previous(e, ctx, &mut idx_out)) + } IndexExpr0::Range(l, r, loc) => IndexExpr0::Range( - wrap_non_matching_in_previous( - l, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - &mut idx_out, - ), - wrap_non_matching_in_previous( - r, - live_source, - live_shape, - other_deps, - source_dim_elements, - iter_ctx, - dims_ctx, - &mut idx_out, - ), + wrap_non_matching_in_previous(l, ctx, &mut idx_out), + wrap_non_matching_in_previous(r, ctx, &mut idx_out), loc, ), other => other, @@ -1231,6 +1311,13 @@ impl PartialEquationError { /// iterated dims + the source's declared dim names + a `DimensionsContext`); /// pass `None` when the live source is scalar (no source subscripts to /// recognize). See [`wrap_non_matching_in_previous`] and [`IteratedDimCtx`]. +/// +/// Test-only since Track A stage 1: production callers now drive the wrap +/// through [`wrap_changed_first_ast`] directly (they need the transformed AST +/// for the post-transform row-pinning / agg-substitution lowerings, not just +/// printed text). This thin text-level wrapper is retained purely as the +/// unit-tested entry point for the per-shape wrap behavior. +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub(crate) fn build_partial_equation_shaped( equation_text: &str, @@ -1253,7 +1340,7 @@ pub(crate) fn build_partial_equation_shaped( .map(|(text, _live_ref)| text) } -/// Like [`build_partial_equation_shaped`], but also returns the *live +/// Like `build_partial_equation_shaped`, but also returns the *live /// source reference* the partial isolates: the single occurrence of /// `live_source` that the PREVIOUS-wrapping transform left un-wrapped, /// with any inner index sub-expressions already PREVIOUS-rewritten. @@ -1271,7 +1358,7 @@ pub(crate) fn build_partial_equation_shaped( /// no left-live `live_source` occurrence at all. /// /// Returns `Err([`PartialEquationError`])` when `equation_text` fails to -/// parse -- see [`build_partial_equation_shaped`] for why this is a loud +/// parse -- see `build_partial_equation_shaped` for why this is a loud /// error rather than a silent lowercased-input fallback (GH #311) -- or /// when the changed-first wrap hit a GH #526 mismatched other-dep /// subscript (`WrapOutcome::other_dep_mismatch`): the collapse would have @@ -1279,6 +1366,9 @@ pub(crate) fn build_partial_equation_shaped( /// `UnfreezablePartial` instead of a silent magnitude error. (Callers that /// can fall back to the changed-last convention route through /// [`shaped_guard_form_text`], which does so for this doom class.) +/// +/// Test-only since Track A stage 1 (see `build_partial_equation_shaped`). +#[cfg(test)] #[allow(clippy::too_many_arguments)] pub(crate) fn build_partial_equation_shaped_with_live_ref( equation_text: &str, @@ -1294,6 +1384,7 @@ pub(crate) fn build_partial_equation_shaped_with_live_ref( deps, live_source, live_shape, + None, source_dim_elements, iter_ctx, dims_ctx, @@ -1309,7 +1400,7 @@ pub(crate) fn build_partial_equation_shaped_with_live_ref( /// [`wrap_non_matching_in_previous`] -- returning the transformed AST (not /// printed text) plus the [`WrapOutcome`] out-channels (the captured live /// reference and the GH #526 mismatch doom flag). The single -/// implementation behind both [`build_partial_equation_shaped_with_live_ref`] +/// implementation behind both `build_partial_equation_shaped_with_live_ref` /// (which prints it) and [`shaped_guard_form_text`] (which doom-checks the /// AST before printing), so the two can never drift on dep filtering, /// parse-failure handling, or the wrap itself. @@ -1319,12 +1410,17 @@ pub(crate) fn build_partial_equation_shaped_with_live_ref( /// Returning the input unchanged would silently produce a non-ceteris- /// paribus "partial" identical to the full equation (link score magnitude /// == 1); fail loudly instead so the caller skips the variable and warns. +/// `live_reducer_text` (Track A stage 1) opts the wrap into holding a +/// designated hoisted reducer subexpression LIVE verbatim (matched by +/// canonical text) instead of an ident -- see [`WrapCtx::live_reducer_text`]. +/// `None` is the ordinary ident-live behavior; every non-agg caller passes it. #[allow(clippy::too_many_arguments)] fn wrap_changed_first_ast( equation_text: &str, deps: &HashSet>, live_source: &Ident, live_shape: &RefShape, + live_reducer_text: Option<&str>, source_dim_elements: &[Vec], iter_ctx: Option<&IteratedDimCtx<'_>>, dims_ctx: Option<&crate::dimensions::DimensionsContext>, @@ -1339,17 +1435,17 @@ fn wrap_changed_first_ast( return Err(PartialEquationError::new(equation_text)); }; - let mut out = WrapOutcome::default(); - let transformed = wrap_non_matching_in_previous( - ast, + let ctx = WrapCtx { live_source, live_shape, - &other_deps, + live_reducer_text, + other_deps: &other_deps, source_dim_elements, iter_ctx, dims_ctx, - &mut out, - ); + }; + let mut out = WrapOutcome::default(); + let transformed = wrap_non_matching_in_previous(ast, &ctx, &mut out); Ok((transformed, out)) } @@ -1763,6 +1859,7 @@ fn shaped_guard_form_text( deps, from, shape, + None, source_dim_elements, iter_ctx, dims_ctx, @@ -1963,7 +2060,7 @@ fn wrap_matching_in_previous(expr: Expr0, target: &Ident) -> Expr0 { /// `(agg - frozen)` in place of `(partial - PREVIOUS(agg))`. /// /// Returns `Err` when `agg_equation_text` does not parse -- same loud-failure -/// contract as [`build_partial_equation_shaped`] (GH #311). +/// contract as `build_partial_equation_shaped` (GH #311). /// `gf_table_ref` is the implicit WITH-LOOKUP wrap (GH #910). `frozen` is a /// full re-evaluation of the agg's own equation, i.e. gf-INPUT units, while /// the numerator subtracts it from the agg's (gf-OUTPUT) current value -- @@ -2189,7 +2286,7 @@ fn pin_iterated_dim_indices(expr: Expr0, dims: &[String], parts: &[String]) -> O /// pin -- a legitimate no-op), and `Err([`PartialEquationError`])` when the /// (already-PREVIOUS-wrapped) partial text fails to re-parse. The latter is /// loud rather than a silent lowercased-input fallback for the same reason -/// as [`build_partial_equation_shaped`] (GH #311): an un-pinned partial may +/// as `build_partial_equation_shaped` (GH #311): an un-pinned partial may /// not even compile, and a silent wrong equation is worse than skipping the /// score with a warning. /// @@ -2358,12 +2455,15 @@ fn subscript_idents_in_expr0( /// `$⁚ltm⁚link_score⁚{from}→{to}[{element}]`, mirroring the arrayed->scalar /// `{from}[{elem}]→{to}` convention from `generate_element_to_scalar_equation`. /// -/// `to_elem_eqn_text` is the target's equation text for this element: the -/// shared A2A body for an `Equation::ApplyToAll` target, or the matching -/// per-element slot's text (or the default slot) for an `Equation::Arrayed` -/// one. `to_deps` is the full dependency set of that equation (computed with -/// the target's AST dimensions so element-name subscripts are not mistaken -/// for variables). `to_deps_to_subscript` is the subset of `to_deps` that +/// `to_elem_eqn_text` is the target's OWN equation text for this element (the +/// hoisted reducers still spelled `SUM(...)`, NOT agg-substituted -- Track A +/// stage 1): the shared A2A body for an `Equation::ApplyToAll` target, or the +/// matching per-element slot's text (or the default slot) for an +/// `Equation::Arrayed` one. `reducer_subst` maps each hoisted reducer's +/// canonical text to its agg name; it is empty for a true scalar `from`. `to_deps` +/// is the full dependency set of that equation (computed with the target's AST +/// dimensions so element-name subscripts are not mistaken for variables), plus +/// the agg names. `to_deps_to_subscript` is the subset of `to_deps` that /// must be element-pinned -- the arrayed deps that share the target's /// dimension (the target self-reference is pinned implicitly via the /// already-subscripted `to[element]` reference the guard form is built @@ -2409,6 +2509,7 @@ pub(crate) fn generate_scalar_to_element_equation( to: &str, element: &str, to_elem_eqn_text: &str, + reducer_subst: &HashMap, to_deps: &HashSet>, to_deps_to_subscript: &HashSet>, source_ref_override: Option<&str>, @@ -2421,19 +2522,27 @@ pub(crate) fn generate_scalar_to_element_equation( let to_q = quote_ident(to); let to_elem = format!("{to_q}[{element}]"); - // The source (a scalar variable, or the bare agg name pre-substitution) - // is referenced bare in `to_elem_eqn_text`, so `RefShape::Bare` holds its - // single occurrence live, `source_dim_elements` is empty (no source - // subscripts to classify), and there is no iterated-dim context. - let partial = build_partial_equation_shaped( + // Composition inverted (Track A stage 1): `to_elem_eqn_text` is the target + // element's OWN equation (reducers still spelled `SUM(...)`). When `from` + // is an arrayed/scalar agg, the reducer that became it is held LIVE by the + // wrap (matched by text via `reducer_subst`); a true scalar `from` (empty + // `reducer_subst`) keeps the ident-live `RefShape::Bare` wrap. Either way + // `source_dim_elements` is empty (no source subscripts to classify) and + // there is no iterated-dim context. The reducer -> agg-name substitution is + // a POST-transform lowering of the wrapped AST, so the later element-pinning + // / `source_pins` passes see the same agg-named text they did before. + let live_reducer_text = live_reducer_text_for_agg(reducer_subst, from); + let (wrapped, _out) = wrap_changed_first_ast( to_elem_eqn_text, to_deps, &from_canonical, &RefShape::Bare, + live_reducer_text, &[], None, dims_ctx, )?; + let partial = print_eqn(&substitute_reducers_in_expr0(wrapped, reducer_subst)); let mut partial = subscript_idents_at_element(&partial, to_deps_to_subscript, element)?; // Pin each mapped ident's references (the live source's numerator // occurrence and any PREVIOUS-wrapped co-agg freezes alike) to its own @@ -2457,292 +2566,26 @@ pub(crate) fn generate_scalar_to_element_equation( Ok(link_score_guard_form(&partial, &to_elem, source_ref)) } -/// Read-only context for [`rewrite_per_element_source_refs`]: everything the -/// walker needs to substitute the live source's subscript indices for one -/// `(site, target element)` instantiation of a `PerElement` link score -/// (GH #525, T6 of the shape-expressiveness design). -struct PerElementRefCtx<'a> { - /// The live source variable (canonical). - from: &'a Ident, - /// The emitting site's per-axis access vector - /// ([`RefShape::PerElement`]'s `axes`). - site_axes: &'a [crate::ltm_agg::AxisRead], - /// The row this `(site, e)` instantiation reads -- BARE element names, - /// one per source axis (parallel to `site_axes`). Occurrences matching - /// `site_axes` are rewritten to exactly these indices so the subsequent - /// changed-first wrap's `FixedIndex(row)` live shape matches them. - row_parts_bare: &'a [String], - /// Element-name lists per source axis (position-strict resolution). - source_dim_elements: &'a [Vec], - /// The source's declared dimensions (for index qualification). - from_dims: &'a [crate::dimensions::Dimension], - /// Target-iterated dim (canonical) -> (element of `e` for that dim, - /// its index within the dim) -- the projection data `e` supplies. - target_elem_by_dim: &'a HashMap, - /// The iterated-dim recognition context (live source's axes + target - /// iterated dims + mapping context). - iter_ctx: &'a IteratedDimCtx<'a>, - dim_ctx: &'a crate::dimensions::DimensionsContext, -} - -/// Qualify one element of one axis (`"nyc"` over `Region` -> -/// `"region\u{B7}nyc"`), via the same defensive rules as -/// [`qualify_element_csv`]. -fn qualify_axis_element(elem: &str, dim: &crate::dimensions::Dimension) -> String { - qualify_element_csv(elem, std::slice::from_ref(dim)) -} - -/// The source row a per-axis access vector reads for one full target -/// element: project the target element onto the `Iterated` axes -/// (slot-remapped through `mapped_element_correspondence` for a -/// positionally-mapped pair -- the correspondence is indexed by TARGET -/// element position and yields the source element the executed simulation -/// reads) and fill `Pinned` axes with their literals. One bare element -/// name per axis, in source-axis order. `None` when an `Iterated` dim is -/// missing from the target projection or the mapped remap is unusable (a -/// mid-edit inconsistency; callers degrade conservatively) -- and for any -/// `Reduced` axis, which the `PerElement` invariant excludes. -/// -/// This is the SINGLE row derivation for the `PerElement` family's -/// emission: the link-score NAME's row (computed by -/// `emit_per_element_link_scores`) and the equation's live-reference row -/// (computed by [`rewrite_per_element_source_refs`]) both come from here, -/// so they cannot disagree. -pub(crate) fn per_element_row_for_target( - axes: &[crate::ltm_agg::AxisRead], - target_elem_by_dim: &HashMap, - dim_ctx: &crate::dimensions::DimensionsContext, -) -> Option> { - use crate::common::CanonicalDimensionName; - use crate::ltm_agg::AxisRead; - axes.iter() - .map(|ax| match ax { - AxisRead::Pinned(e) => Some(e.clone()), - AxisRead::Iterated { dim, source_dim } => { - let (elem, idx) = target_elem_by_dim.get(dim)?; - if dim == source_dim { - Some(elem.clone()) - } else { - let corr = dim_ctx.mapped_element_correspondence( - &CanonicalDimensionName::from_raw(dim), - &CanonicalDimensionName::from_raw(source_dim), - )?; - corr.get(*idx).map(|e| e.as_str().to_string()) - } - } - AxisRead::Reduced { .. } => None, - }) - .collect() -} - -/// Rewrite every reference to the live source inside a `PerElement` link -/// score's target body so the subsequent changed-first wrap and the final -/// scalar equation are well-formed for one `(site, target element)` -/// instantiation -- the index-substitution mechanism of `pin_body_to_row` -/// (the GH #744 reducer-body machinery) lifted to target-equation bodies -/// (T6 of the shape-expressiveness design): -/// -/// - an occurrence whose per-axis classification EQUALS the emitting -/// site's `axes` is rewritten to the row's BARE element indices -- the -/// changed-first wrap then holds it live via its `FixedIndex(row)` -/// live shape (and only it: every other source occurrence ends up -/// shaped differently); -/// - any other fully-classifiable occurrence (a different `PerElement` -/// shape, an all-`Iterated` Bare-shaped subscript, an all-`Pinned` -/// literal subscript) is rewritten to ITS OWN row for this target -/// element, QUALIFIED (`region\u{B7}a`) so the wrap's `PREVIOUS(...)` -/// freeze compiles to a direct LoadPrev in the scalar fragment; -/// - a BARE `Var` occurrence of the source (the mixed `Bare`+`PerElement` -/// edge's other site) is pinned to the target element's projection onto -/// the source's own axes when every axis resolves (same-element -/// semantics), qualified -- else left for the wrap's conservative -/// freeze; -/// - a partially-classifiable subscript (a wildcard slice, a dynamic -/// index) gets only its resolvable iterated-dim indices substituted -/// (qualified; meaning-preserving -- the iterated dim IS that element -/// in this slot), leaving the rest for the wrap's conservative -/// handling. -/// -/// Inside `PREVIOUS(...)`/`INIT(...)` calls the live-match rewrite is -/// suppressed (`force_qualified`): the contents are already lagged/frozen -/// and the wrap never recurses into them, so a bare-row rewrite there -/// could never be the live reference -- qualified substitution keeps the -/// frozen read compiling to a direct slot. -fn rewrite_per_element_source_refs( - expr: Expr0, - ctx: &PerElementRefCtx<'_>, - force_qualified: bool, -) -> Expr0 { - let qualify_row = |row: &[String]| -> Vec { - row.iter() - .zip(ctx.from_dims) - .map(|(part, dim)| { - IndexExpr0::Expr(Expr0::Var( - RawIdent::new_from_str(&qualify_axis_element(part, dim)), - crate::ast::Loc::default(), - )) - }) - .collect() - }; - match expr { - Expr0::Const(..) => expr, - Expr0::Var(ref ident, loc) => { - if &Ident::::new(ident.as_str()) != ctx.from { - return expr; - } - // Same-element pin of a bare source reference: each axis reads - // the target element's coordinate for that axis's own dim. - let bare_axes: Vec = ctx - .from_dims - .iter() - .map(|d| crate::ltm_agg::AxisRead::Iterated { - dim: d.name().to_string(), - source_dim: d.name().to_string(), - }) - .collect(); - match per_element_row_for_target(&bare_axes, ctx.target_elem_by_dim, ctx.dim_ctx) { - Some(row) => Expr0::Subscript(ident.clone(), qualify_row(&row), loc), - None => expr, - } - } - Expr0::Subscript(ident, indices, loc) => { - if &Ident::::new(ident.as_str()) != ctx.from { - // Another variable's subscript: recurse into expression - // indices (a nested source reference can hide there). - let indices = - indices - .into_iter() - .map(|idx| match idx { - IndexExpr0::Expr(e) => IndexExpr0::Expr( - rewrite_per_element_source_refs(e, ctx, force_qualified), - ), - other => other, - }) - .collect(); - return Expr0::Subscript(ident, indices, loc); - } - if let Some(occ_axes) = - classify_expr0_per_element_axes(&indices, ctx.source_dim_elements, ctx.iter_ctx) - { - if !force_qualified && occ_axes == ctx.site_axes { - // The emitting site's own shape: rewrite to the row's - // bare elements so the changed-first wrap's - // `FixedIndex(row)` live shape matches it. - let indices = ctx - .row_parts_bare - .iter() - .map(|p| { - IndexExpr0::Expr(Expr0::Var( - RawIdent::new_from_str(p), - crate::ast::Loc::default(), - )) - }) - .collect(); - return Expr0::Subscript(ident, indices, loc); - } - if let Some(row) = - per_element_row_for_target(&occ_axes, ctx.target_elem_by_dim, ctx.dim_ctx) - { - return Expr0::Subscript(ident, qualify_row(&row), loc); - } - return Expr0::Subscript(ident, indices, loc); - } - // Partially classifiable: substitute only the resolvable - // iterated-dim indices (qualified), leave the rest (wildcards, - // literals, dynamic expressions) for the wrap's conservative - // handling. - let indices = indices - .into_iter() - .enumerate() - .map(|(i, idx)| { - let substituted = match &idx { - IndexExpr0::Expr(Expr0::Var(name, _)) => { - let d = canonicalize(name.as_str()).into_owned(); - if i < ctx.from_dims.len() - && ctx.iter_ctx.target_iterated_dims.contains(&d) - && expr0_iterated_axis_lines_up( - &d, - i, - ctx.source_dim_elements, - ctx.iter_ctx, - ) - { - let ax = crate::ltm_agg::AxisRead::Iterated { - dim: d, - source_dim: ctx.iter_ctx.source_dim_names[i].clone(), - }; - per_element_row_for_target( - std::slice::from_ref(&ax), - ctx.target_elem_by_dim, - ctx.dim_ctx, - ) - .map(|row| qualify_axis_element(&row[0], &ctx.from_dims[i])) - } else { - None - } - } - _ => None, - }; - match substituted { - Some(part) => IndexExpr0::Expr(Expr0::Var( - RawIdent::new_from_str(&part), - crate::ast::Loc::default(), - )), - None => idx, - } - }) - .collect(); - Expr0::Subscript(ident, indices, loc) - } - Expr0::App(UntypedBuiltinFn(name, args), loc) => { - let lagged = name.eq_ignore_ascii_case("previous") || name.eq_ignore_ascii_case("init"); - let args = args - .into_iter() - .map(|a| rewrite_per_element_source_refs(a, ctx, force_qualified || lagged)) - .collect(); - Expr0::App(UntypedBuiltinFn(name, args), loc) - } - Expr0::Op1(op, inner, loc) => Expr0::Op1( - op, - Box::new(rewrite_per_element_source_refs( - *inner, - ctx, - force_qualified, - )), - loc, - ), - Expr0::Op2(op, l, r, loc) => Expr0::Op2( - op, - Box::new(rewrite_per_element_source_refs(*l, ctx, force_qualified)), - Box::new(rewrite_per_element_source_refs(*r, ctx, force_qualified)), - loc, - ), - Expr0::If(c, t, f, loc) => Expr0::If( - Box::new(rewrite_per_element_source_refs(*c, ctx, force_qualified)), - Box::new(rewrite_per_element_source_refs(*t, ctx, force_qualified)), - Box::new(rewrite_per_element_source_refs(*f, ctx, force_qualified)), - loc, - ), - } -} - /// Generate the per-(row, full-target-element) scalar link-score equation /// for one `PerElement` reference site (GH #525, T6 of the /// shape-expressiveness design): the partial of target element /// `element_qualified`'s equation w.r.t. the live source's `site_axes` -/// occurrence, with -/// -/// - the live occurrence rewritten to the concrete row subscript -/// `{from}[{row}]` (a real `Expr0::Subscript`, never `SUM(...)`-wrapped) -/// and held live by the changed-first wrap's internal `FixedIndex(row)` -/// shape, -/// - every OTHER source occurrence pinned to ITS row for this element and -/// frozen at `PREVIOUS` (each is attributed by its own link score: +/// occurrence. The composition is `wrap(own equation) then row-pin` (Track A +/// stage 1): the ceteris-paribus wrap runs on the target element's OWN +/// equation text with the site's actual `PerElement { site_axes }` shape held +/// live, and the row-pinning is a POST-transform lowering +/// ([`rewrite_per_element_source_refs`]) of the wrapped AST. The result: +/// +/// - the live occurrence (held live by the wrap because its shape equals +/// `site_axes`) lowered to the concrete row subscript `{from}[{row}]` (a +/// real `Expr0::Subscript`, never `SUM(...)`-wrapped), +/// - every OTHER source occurrence (frozen at `PREVIOUS` by the wrap) lowered +/// to ITS row for this element (each is attributed by its own link score: /// another `PerElement` site's scalar, the Bare A2A score of a mixed /// edge, a `FixedIndex` site's per-element score), /// - the target's other arrayed deps element-pinned via /// [`subscript_idents_at_element`] (`to_deps_to_subscript` must NOT -/// contain the source -- its pinning is the rewrite pass's job), and +/// contain the source -- its pinning is the lowering's job), and /// - the guard form's target/source references pinned to /// `to[{element}]` / `{from}[{row}]`. /// @@ -2779,15 +2622,13 @@ pub(crate) fn generate_per_element_link_equation( source_dim_names: &source_dim_names, target_iterated_dims, dim_ctx: Some(dims_ctx), - // The changed-first wrap below runs with `iter_ctx: None` (the - // source's iterated subscripts are rewritten away first), so the - // GH #526 other-dep verdict never consults this ctx; no dep dims - // to thread. + // The changed-first wrap below runs with this ctx `Some` (the live + // source's `PerElement` shape is recognized against the target's + // iterated dims), but the `PerElement` live shape suppresses the + // GH #526 other-dep collapse entirely (see `wrap_non_matching_in_previous`), + // so the verdict's `dep_dims` are never consulted; none to thread. dep_dims: None, }; - let Ok(Some(ast)) = Expr0::new(to_elem_eqn_text, LexerType::Equation) else { - return Err(PartialEquationError::new(to_elem_eqn_text)); - }; let ref_ctx = PerElementRefCtx { from: &from_canonical, site_axes, @@ -2798,24 +2639,45 @@ pub(crate) fn generate_per_element_link_equation( iter_ctx: &iter_ctx, dim_ctx: dims_ctx, }; - let rewritten = rewrite_per_element_source_refs(ast, &ref_ctx, false); - let rewritten_text = print_eqn(&rewritten); - // The changed-first wrap: the rewritten live occurrence is the only one - // matching `FixedIndex(row)` (every other source occurrence was pinned - // QUALIFIED, which classifies `DynamicIndex` and freezes). No - // iterated-dim context: the source's iterated subscripts no longer - // exist (rewritten above), and other deps' iterated indices are pinned - // by `subscript_idents_at_element`'s dimension-name pinning below. - let live_shape = RefShape::FixedIndex(row_parts_bare.to_vec()); - let partial = build_partial_equation_shaped( - &rewritten_text, + // Invert the composition (Track A stage 1): run the ceteris-paribus wrap + // on the target element's OWN equation, holding the site's ACTUAL + // `PerElement` shape live, and make the row-pinning a POST-transform + // lowering of the already-wrapped AST. Historically this pinned every + // source ref to a concrete row FIRST and then wrapped a synthesized + // `FixedIndex(row)`-shaped *derived* text -- text that no occurrence stream + // describes, so the occurrence IR (which enumerates the target's OWN + // equation) could not drive the wrap. Running the wrap on the own text and + // lowering afterward keeps the byte-identical output while making the wrap + // occurrence-addressable for stage 2. The `PerElement` live shape + // suppresses the GH #526 other-dep collapse (an iterated other-dep like + // `w[Age]` keeps its subscript so the post-transform per-element pin can + // resolve each dimension-name index -- collapsing to bare would let the + // full-tuple pin over-subscript a subset-dims dep), so the mismatch doom + // can never fire here; the `out.other_dep_mismatch` check below is retained + // defensively to preserve `wrap_changed_first_ast`'s doom contract. + let live_shape = RefShape::PerElement { + axes: site_axes.to_vec(), + }; + let (wrapped, out) = wrap_changed_first_ast( + to_elem_eqn_text, to_deps, &from_canonical, &live_shape, - &source_dim_elements, None, + &source_dim_elements, + Some(&iter_ctx), Some(dims_ctx), )?; + if out.other_dep_mismatch { + return Err(PartialEquationError::unfreezable(to_elem_eqn_text)); + } + // POST-transform row-pinning lowering: rewrite the wrapped AST's live and + // frozen source occurrences (including those the wrap moved inside + // `PREVIOUS(...)`) to their concrete per-element subscripts -- the live + // occurrence to its bare row (so it re-prints as the historical + // `from[]`), every other occurrence to its own qualified row. + let lowered = rewrite_per_element_source_refs(wrapped, &ref_ctx, false); + let partial = print_eqn(&lowered); let mut partial = subscript_idents_at_element(&partial, to_deps_to_subscript, element_qualified)?; if let Some(table_ref) = gf_table_ref { @@ -2834,24 +2696,35 @@ pub(crate) fn generate_per_element_link_equation( /// Generate the `agg → scalar-target` link-score equation: the partial of /// `to`'s (scalar) equation w.r.t. the aggregate node `agg_name` held live, -/// everything else PREVIOUS. `to_eqn_text` is the target's equation text with -/// every hoisted reducer subexpression already substituted by its agg name -/// (so the agg appears where `SUM(...)` was); `to_deps` is the (over- -/// approximating is fine) dependency set of that substituted text. The result -/// is `Equation::Scalar`-shaped text, named `$⁚ltm⁚link_score⁚{agg}→{to}`. +/// everything else PREVIOUS. The result is `Equation::Scalar`-shaped text, +/// named `$⁚ltm⁚link_score⁚{agg}→{to}`. +/// +/// Composition inverted (Track A stage 1): `to_own_eqn_text` is `to`'s OWN +/// equation text (hoisted reducers still spelled `SUM(...)`), and +/// `reducer_subst` maps each hoisted reducer's canonical text to its agg name. +/// The ceteris-paribus wrap runs on that own text with the reducer that became +/// `agg_name` held LIVE verbatim (matched by text; every co-reducer freezes +/// whole via the GH #517 path), and the agg substitution is a POST-transform +/// lowering ([`substitute_reducers_in_expr0`]) of the wrapped AST -- the +/// held-live reducer to a bare agg name, each frozen `PREVIOUS(SUM(..))` to +/// `PREVIOUS(agg)`. This keeps the wrap on the target's own equation (so the +/// occurrence IR can drive it in stage 2) with byte-identical output. `to_deps` +/// is the (over-approximating is fine) dependency set that includes `to`'s own +/// reducer-source vars and the agg names. /// /// For an *arrayed* target the per-target-element form is produced by /// [`generate_scalar_to_element_equation`] instead (with `from = agg_name`). /// /// `gf_table_ref` is the target's implicit WITH-LOOKUP table reference /// ([`with_lookup_table_ref`], `None` when the target applies no gf): the -/// partial re-evaluates `to_eqn_text`, which is the gf's *input*, so it must -/// be fed through the table to be commensurable with the `PREVIOUS(to)` -/// anchor the guard form subtracts (GH #910). +/// partial re-evaluates the gf's *input*, so it must be fed through the table +/// to be commensurable with the `PREVIOUS(to)` anchor the guard form subtracts +/// (GH #910). pub(crate) fn generate_agg_to_scalar_target_equation( agg_name: &str, to_name: &str, - to_eqn_text: &str, + to_own_eqn_text: &str, + reducer_subst: &HashMap, to_deps: &HashSet>, dims_ctx: Option<&crate::dimensions::DimensionsContext>, gf_table_ref: Option<&str>, @@ -2859,125 +2732,43 @@ pub(crate) fn generate_agg_to_scalar_target_equation( let agg_canonical = Ident::new(agg_name); let agg_q = quote_ident(agg_name); let to_q = quote_ident(to_name); - // The agg node is a scalar -- referenced bare, no iterated-dim context. - let mut partial = build_partial_equation_shaped( - to_eqn_text, + // The reducer subexpression that was hoisted into `agg_name` -- held live + // verbatim by the wrap. `None` (agg not in `reducer_subst`) degrades to the + // ident-live wrap on `agg_canonical`, which never matches the own text and + // so freezes everything (defensive; not a production shape). The agg node + // is a scalar -- referenced bare, no iterated-dim context. + let live_reducer_text = live_reducer_text_for_agg(reducer_subst, agg_name); + let (wrapped, _out) = wrap_changed_first_ast( + to_own_eqn_text, to_deps, &agg_canonical, &RefShape::Bare, + live_reducer_text, &[], None, dims_ctx, )?; + let mut partial = print_eqn(&substitute_reducers_in_expr0(wrapped, reducer_subst)); if let Some(table_ref) = gf_table_ref { partial = format!("LOOKUP({table_ref}, {partial})"); } Ok(link_score_guard_form(&partial, &to_q, &agg_q)) } -/// Substitute each recognized reducer subexpression in `equation_text` with a -/// (quoted) reference to its aggregate node. -/// -/// `reducers` maps the canonical reducer-subexpression text (exactly as -/// `crate::patch::expr2_to_string` / `print_eqn` renders it -- lowercased, -/// whitespace-normalized) to the agg node's name. `equation_text` is parsed -/// to `Expr0`, and any subexpression of it whose `print_eqn` equals one of -/// those keys is replaced by a `Var(agg_name)` node, then the whole tree is -/// re-printed. The match is on the parsed AST subtree, not a substring of the -/// text, so a reducer text that is a textual prefix of a *different* reducer -/// subexpression (`sum(p[*])` vs `sum(p[*] + 1)`) is never falsely matched. -/// -/// Returns `Err([`PartialEquationError`])` when `equation_text` does not parse -/// (a genuine parse error, or an empty/whitespace equation that yields no AST) -/// *and there are reducers to substitute*: with no AST there is no reducer -/// subexpression to replace, so returning the input unchanged would let the -/// inline reducer survive into the `agg → target` partial -- a partial that -/// references the live reducer instead of the hoisted aggregate node, a -/// wrong-but-clean-compiling link score (the agg-substitution-omission sibling -/// of the GH #311 PREVIOUS-omission hazard; GH #661). The db-bearing caller -/// converts the error into a `Warning` (via `emit_ltm_partial_equation_warning`) -/// and skips the variable. The failure is effectively unreachable in production -/// (the input is a `print_eqn` re-print of an already-parsed AST), so this is -/// defense-in-depth. -/// -/// The empty-`reducers` case is a pure pass-through that never parses (there -/// is nothing to substitute), so it returns `Ok` with the text unchanged even -/// for otherwise-unparseable input. -pub(crate) fn substitute_reducers_in_equation( - equation_text: &str, - reducers: &HashMap, -) -> Result { - if reducers.is_empty() { - return Ok(equation_text.to_string()); - } - let Ok(Some(ast)) = Expr0::new(equation_text, LexerType::Equation) else { - return Err(PartialEquationError::new(equation_text)); - }; - Ok(print_eqn(&substitute_reducers_in_expr0(ast, reducers))) -} - -fn substitute_reducers_in_expr0(expr: Expr0, reducers: &HashMap) -> Expr0 { - // A whole-subtree match wins before descending: a reducer App is opaque - // -- once it matches an agg, we don't recurse into its (now-irrelevant) - // argument. - if let Some(agg_name) = reducers.get(&print_eqn(&expr)) { - return Expr0::Var( - crate::common::RawIdent::new_from_str(agg_name), - crate::ast::Loc::default(), - ); - } - match expr { - Expr0::Const(..) | Expr0::Var(..) => expr, - Expr0::Subscript(ident, indices, loc) => { - // A reducer can appear as (or inside) a subscript index expression - // -- `stock[SUM(idx[*])]` -- and `walk_subexpr_for_aggs` hoists it - // into a synthetic agg by descending into `IndexExpr2::Expr` / - // `IndexExpr2::Range`, so the substituter must mirror that descent. - // Wildcard / star-range / `@N` indices carry no `Expr0`, so they - // pass through unchanged. - let indices = indices - .into_iter() - .map(|idx| match idx { - IndexExpr0::Expr(e) => { - IndexExpr0::Expr(substitute_reducers_in_expr0(e, reducers)) - } - IndexExpr0::Range(l, r, loc) => IndexExpr0::Range( - substitute_reducers_in_expr0(l, reducers), - substitute_reducers_in_expr0(r, reducers), - loc, - ), - IndexExpr0::Wildcard(_) - | IndexExpr0::StarRange(_, _) - | IndexExpr0::DimPosition(_, _) => idx, - }) - .collect(); - Expr0::Subscript(ident, indices, loc) - } - Expr0::App(UntypedBuiltinFn(name, args), loc) => { - let args = args - .into_iter() - .map(|a| substitute_reducers_in_expr0(a, reducers)) - .collect(); - Expr0::App(UntypedBuiltinFn(name, args), loc) - } - Expr0::Op1(op, inner, loc) => Expr0::Op1( - op, - Box::new(substitute_reducers_in_expr0(*inner, reducers)), - loc, - ), - Expr0::Op2(op, lhs, rhs, loc) => Expr0::Op2( - op, - Box::new(substitute_reducers_in_expr0(*lhs, reducers)), - Box::new(substitute_reducers_in_expr0(*rhs, reducers)), - loc, - ), - Expr0::If(cond, then_e, else_e, loc) => Expr0::If( - Box::new(substitute_reducers_in_expr0(*cond, reducers)), - Box::new(substitute_reducers_in_expr0(*then_e, reducers)), - Box::new(substitute_reducers_in_expr0(*else_e, reducers)), - loc, - ), - } +/// The canonical text of the hoisted reducer that became `agg_name`, for the +/// wrap's `live_reducer_text` channel (Track A stage 1). `reducer_subst` maps +/// reducer text -> agg name; an agg is hoisted from exactly one canonical +/// reducer text, so at most one entry matches. `None` when `agg_name` is not a +/// key's value (an empty `reducer_subst`, or a true scalar `from`), which keeps +/// the historical ident-live wrap. +fn live_reducer_text_for_agg<'a>( + reducer_subst: &'a HashMap, + agg_name: &str, +) -> Option<&'a str> { + reducer_subst + .iter() + .find(|(_, agg)| agg.as_str() == agg_name) + .map(|(text, _)| text.as_str()) } /// Quote an identifier for use in an equation string. @@ -3573,7 +3364,7 @@ fn arrayed_target_dim_names( /// /// For each `(element, expr)` slot in the target's per-element map, the /// slot equation is the standard link-score guard form ([`link_score_guard_form`]) -/// whose `{partial}` is [`build_partial_equation_shaped`] applied to *that +/// whose `{partial}` is `build_partial_equation_shaped` applied to *that /// element's own equation text* with `live_source = from` and `live_shape = /// shape`. So the cross-element partial derived from /// `mp[NYC] = (pop[NYC] - pop[Boston]) * 0.01` keeps `pop[NYC]` live and diff --git a/src/simlin-engine/src/ltm_augment_post_transform.rs b/src/simlin-engine/src/ltm_augment_post_transform.rs new file mode 100644 index 000000000..3f14d28f1 --- /dev/null +++ b/src/simlin-engine/src/ltm_augment_post_transform.rs @@ -0,0 +1,455 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! Post-transform lowering for LTM link-score equations (Track A stage 1). +//! +//! The `agg → target` and `PerElement` link-score generators in `ltm_augment` +//! were inverted by the "transform-first" restructuring: the ceteris-paribus +//! wrap (`wrap_changed_first_ast`) now runs on the target's OWN equation text +//! -- hoisted reducers still spelled `SUM(...)`, the live source's occurrence +//! held at its actual shape -- and the concrete-form rewrite runs AFTERWARD, on +//! the already-wrapped `Expr0`. This module owns those post-transform lowerings: +//! +//! - [`substitute_reducers_in_expr0`] rewrites each hoisted reducer subtree in +//! the wrapped AST to its synthetic aggregate node's bare name (the held-live +//! reducer to `agg`, each frozen `PREVIOUS(SUM(..))` to `PREVIOUS(agg)`); +//! - [`rewrite_per_element_source_refs`] row-pins every live/frozen source +//! occurrence of a `PerElement` edge to its concrete per-element subscript, +//! supported by [`PerElementRefCtx`], [`per_element_row_for_target`] (the +//! single row derivation the link-score NAME and equation both consume), and +//! [`qualify_axis_element`]. +//! +//! Running the wrap on the own text and lowering afterward keeps the emitted +//! equation byte-identical while making the wrap occurrence-addressable -- it is +//! now keyed on the target's OWN occurrence stream. That is the whole point of +//! the inversion: stage 2/3 can retarget the wrap onto the `db::ltm_ir` +//! occurrence IR without touching these lowerings, because they deliberately do +//! NOT re-classify (each is a pure text-keyed / shape-keyed AST rewrite). See +//! the generators in `ltm_augment` -- `generate_agg_to_scalar_target_equation`, +//! `generate_scalar_to_element_equation`, and `generate_per_element_link_equation` +//! -- for the wrap-then-lower composition that calls these. +//! +//! Split out of `ltm_augment.rs` to keep that file under the project +//! line-count lint; included via `#[path]`, so `super::*` resolves the parent's +//! private items. + +use std::collections::HashMap; + +use crate::ast::{Expr0, IndexExpr0, print_eqn}; +use crate::builtins::UntypedBuiltinFn; +use crate::canonicalize; +use crate::common::{Canonical, Ident, RawIdent}; + +use super::{ + IteratedDimCtx, classify_expr0_per_element_axes, expr0_iterated_axis_lines_up, + qualify_element_csv, +}; + +#[cfg(test)] +use super::PartialEquationError; +#[cfg(test)] +use crate::lexer::LexerType; + +/// Read-only context for [`rewrite_per_element_source_refs`]: everything the +/// walker needs to substitute the live source's subscript indices for one +/// `(site, target element)` instantiation of a `PerElement` link score +/// (GH #525, T6 of the shape-expressiveness design). +pub(super) struct PerElementRefCtx<'a> { + /// The live source variable (canonical). + pub(super) from: &'a Ident, + /// The emitting site's per-axis access vector + /// ([`RefShape::PerElement`]'s `axes`). + pub(super) site_axes: &'a [crate::ltm_agg::AxisRead], + /// The row this `(site, e)` instantiation reads -- BARE element names, + /// one per source axis (parallel to `site_axes`). The wrap already held + /// the `site_axes`-shaped occurrence live (its shape equals `site_axes`); + /// this lowering rewrites that live occurrence to exactly these bare + /// indices so it re-prints as the historical `from[]`. + pub(super) row_parts_bare: &'a [String], + /// Element-name lists per source axis (position-strict resolution). + pub(super) source_dim_elements: &'a [Vec], + /// The source's declared dimensions (for index qualification). + pub(super) from_dims: &'a [crate::dimensions::Dimension], + /// Target-iterated dim (canonical) -> (element of `e` for that dim, + /// its index within the dim) -- the projection data `e` supplies. + pub(super) target_elem_by_dim: &'a HashMap, + /// The iterated-dim recognition context (live source's axes + target + /// iterated dims + mapping context). + pub(super) iter_ctx: &'a IteratedDimCtx<'a>, + pub(super) dim_ctx: &'a crate::dimensions::DimensionsContext, +} + +/// Qualify one element of one axis (`"nyc"` over `Region` -> +/// `"region\u{B7}nyc"`), via the same defensive rules as +/// [`qualify_element_csv`]. +pub(super) fn qualify_axis_element(elem: &str, dim: &crate::dimensions::Dimension) -> String { + qualify_element_csv(elem, std::slice::from_ref(dim)) +} + +/// The source row a per-axis access vector reads for one full target +/// element: project the target element onto the `Iterated` axes +/// (slot-remapped through `mapped_element_correspondence` for a +/// positionally-mapped pair -- the correspondence is indexed by TARGET +/// element position and yields the source element the executed simulation +/// reads) and fill `Pinned` axes with their literals. One bare element +/// name per axis, in source-axis order. `None` when an `Iterated` dim is +/// missing from the target projection or the mapped remap is unusable (a +/// mid-edit inconsistency; callers degrade conservatively) -- and for any +/// `Reduced` axis, which the `PerElement` invariant excludes. +/// +/// This is the SINGLE row derivation for the `PerElement` family's +/// emission: the link-score NAME's row (computed by +/// `emit_per_element_link_scores`) and the equation's live-reference row +/// (computed by [`rewrite_per_element_source_refs`]) both come from here, +/// so they cannot disagree. +pub(crate) fn per_element_row_for_target( + axes: &[crate::ltm_agg::AxisRead], + target_elem_by_dim: &HashMap, + dim_ctx: &crate::dimensions::DimensionsContext, +) -> Option> { + use crate::common::CanonicalDimensionName; + use crate::ltm_agg::AxisRead; + axes.iter() + .map(|ax| match ax { + AxisRead::Pinned(e) => Some(e.clone()), + AxisRead::Iterated { dim, source_dim } => { + let (elem, idx) = target_elem_by_dim.get(dim)?; + if dim == source_dim { + Some(elem.clone()) + } else { + let corr = dim_ctx.mapped_element_correspondence( + &CanonicalDimensionName::from_raw(dim), + &CanonicalDimensionName::from_raw(source_dim), + )?; + corr.get(*idx).map(|e| e.as_str().to_string()) + } + } + AxisRead::Reduced { .. } => None, + }) + .collect() +} + +/// Row-pin every reference to the live source inside a `PerElement` link +/// score's target body so the final scalar equation is well-formed for one +/// `(site, target element)` instantiation -- the index-substitution mechanism +/// of `pin_body_to_row` (the GH #744 reducer-body machinery) lifted to +/// target-equation bodies (T6 of the shape-expressiveness design). +/// +/// This runs as a POST-transform lowering of the already-wrapped AST (Track A +/// stage 1): the ceteris-paribus wrap (`wrap_changed_first_ast` with +/// `PerElement { site_axes }` held live) already held the emitting site's +/// occurrence live and froze every other reference at `PREVIOUS`; this pass +/// then substitutes each source occurrence's iterated-dim indices for concrete +/// elements. The distinction the wrap already encoded -- live occurrence not +/// nested in a wrap-inserted `PREVIOUS`, frozen occurrences nested inside one +/// -- lines up with the `force_qualified` flag below, so the lowering reads +/// "live vs frozen" off `PREVIOUS`-nesting rather than re-deciding it: +/// +/// - an occurrence whose per-axis classification EQUALS the emitting site's +/// `axes` (and is not `force_qualified`, i.e. the wrap left it live) is +/// rewritten to the row's BARE element indices, so it re-prints as the +/// historical `from[]`; +/// - any other fully-classifiable occurrence (a different `PerElement` +/// shape, an all-`Iterated` Bare-shaped subscript, an all-`Pinned` +/// literal subscript) is rewritten to ITS OWN row for this target +/// element, QUALIFIED (`region\u{B7}a`) so its wrap-inserted `PREVIOUS(...)` +/// freeze compiles to a direct LoadPrev in the scalar fragment; +/// - a BARE `Var` occurrence of the source (the mixed `Bare`+`PerElement` +/// edge's other site) is pinned to the target element's projection onto +/// the source's own axes when every axis resolves (same-element +/// semantics), qualified -- else left as the wrap's conservative freeze; +/// - a partially-classifiable subscript (a wildcard slice, a dynamic +/// index) gets only its resolvable iterated-dim indices substituted +/// (qualified; meaning-preserving -- the iterated dim IS that element +/// in this slot), leaving the rest as the wrap left it. +/// +/// Inside `PREVIOUS(...)`/`INIT(...)` calls (both the wrap-inserted freezes and +/// any already in the source equation) the live-match rewrite is suppressed +/// (`force_qualified`): the contents are lagged/frozen and could never be the +/// live reference -- qualified substitution keeps the frozen read compiling to +/// a direct slot. +pub(super) fn rewrite_per_element_source_refs( + expr: Expr0, + ctx: &PerElementRefCtx<'_>, + force_qualified: bool, +) -> Expr0 { + let qualify_row = |row: &[String]| -> Vec { + row.iter() + .zip(ctx.from_dims) + .map(|(part, dim)| { + IndexExpr0::Expr(Expr0::Var( + RawIdent::new_from_str(&qualify_axis_element(part, dim)), + crate::ast::Loc::default(), + )) + }) + .collect() + }; + match expr { + Expr0::Const(..) => expr, + Expr0::Var(ref ident, loc) => { + if &Ident::::new(ident.as_str()) != ctx.from { + return expr; + } + // Same-element pin of a bare source reference: each axis reads + // the target element's coordinate for that axis's own dim. + let bare_axes: Vec = ctx + .from_dims + .iter() + .map(|d| crate::ltm_agg::AxisRead::Iterated { + dim: d.name().to_string(), + source_dim: d.name().to_string(), + }) + .collect(); + match per_element_row_for_target(&bare_axes, ctx.target_elem_by_dim, ctx.dim_ctx) { + Some(row) => Expr0::Subscript(ident.clone(), qualify_row(&row), loc), + None => expr, + } + } + Expr0::Subscript(ident, indices, loc) => { + if &Ident::::new(ident.as_str()) != ctx.from { + // Another variable's subscript: recurse into expression + // indices (a nested source reference can hide there). + let indices = + indices + .into_iter() + .map(|idx| match idx { + IndexExpr0::Expr(e) => IndexExpr0::Expr( + rewrite_per_element_source_refs(e, ctx, force_qualified), + ), + other => other, + }) + .collect(); + return Expr0::Subscript(ident, indices, loc); + } + if let Some(occ_axes) = + classify_expr0_per_element_axes(&indices, ctx.source_dim_elements, ctx.iter_ctx) + { + if !force_qualified && occ_axes == ctx.site_axes { + // The emitting site's own shape: lower the occurrence + // (already held live or frozen by the preceding + // ceteris-paribus wrap) to the concrete row this score + // targets, spelled as bare elements. + let indices = ctx + .row_parts_bare + .iter() + .map(|p| { + IndexExpr0::Expr(Expr0::Var( + RawIdent::new_from_str(p), + crate::ast::Loc::default(), + )) + }) + .collect(); + return Expr0::Subscript(ident, indices, loc); + } + if let Some(row) = + per_element_row_for_target(&occ_axes, ctx.target_elem_by_dim, ctx.dim_ctx) + { + return Expr0::Subscript(ident, qualify_row(&row), loc); + } + return Expr0::Subscript(ident, indices, loc); + } + // Partially classifiable: substitute only the resolvable + // iterated-dim indices (qualified), leave the rest (wildcards, + // literals, dynamic expressions) for the wrap's conservative + // handling. + let indices = indices + .into_iter() + .enumerate() + .map(|(i, idx)| { + let substituted = match &idx { + IndexExpr0::Expr(Expr0::Var(name, _)) => { + let d = canonicalize(name.as_str()).into_owned(); + if i < ctx.from_dims.len() + && ctx.iter_ctx.target_iterated_dims.contains(&d) + && expr0_iterated_axis_lines_up( + &d, + i, + ctx.source_dim_elements, + ctx.iter_ctx, + ) + { + let ax = crate::ltm_agg::AxisRead::Iterated { + dim: d, + source_dim: ctx.iter_ctx.source_dim_names[i].clone(), + }; + per_element_row_for_target( + std::slice::from_ref(&ax), + ctx.target_elem_by_dim, + ctx.dim_ctx, + ) + .map(|row| qualify_axis_element(&row[0], &ctx.from_dims[i])) + } else { + None + } + } + _ => None, + }; + match substituted { + Some(part) => IndexExpr0::Expr(Expr0::Var( + RawIdent::new_from_str(&part), + crate::ast::Loc::default(), + )), + None => idx, + } + }) + .collect(); + Expr0::Subscript(ident, indices, loc) + } + Expr0::App(UntypedBuiltinFn(name, args), loc) => { + let lagged = name.eq_ignore_ascii_case("previous") || name.eq_ignore_ascii_case("init"); + let args = args + .into_iter() + .map(|a| rewrite_per_element_source_refs(a, ctx, force_qualified || lagged)) + .collect(); + Expr0::App(UntypedBuiltinFn(name, args), loc) + } + Expr0::Op1(op, inner, loc) => Expr0::Op1( + op, + Box::new(rewrite_per_element_source_refs( + *inner, + ctx, + force_qualified, + )), + loc, + ), + Expr0::Op2(op, l, r, loc) => Expr0::Op2( + op, + Box::new(rewrite_per_element_source_refs(*l, ctx, force_qualified)), + Box::new(rewrite_per_element_source_refs(*r, ctx, force_qualified)), + loc, + ), + Expr0::If(c, t, f, loc) => Expr0::If( + Box::new(rewrite_per_element_source_refs(*c, ctx, force_qualified)), + Box::new(rewrite_per_element_source_refs(*t, ctx, force_qualified)), + Box::new(rewrite_per_element_source_refs(*f, ctx, force_qualified)), + loc, + ), + } +} + +/// Substitute each recognized reducer subexpression in `equation_text` with a +/// (quoted) reference to its aggregate node. +/// +/// `reducers` maps the canonical reducer-subexpression text (exactly as +/// `crate::patch::expr2_to_string` / `print_eqn` renders it -- lowercased, +/// whitespace-normalized) to the agg node's name. `equation_text` is parsed +/// to `Expr0`, and any subexpression of it whose `print_eqn` equals one of +/// those keys is replaced by a `Var(agg_name)` node, then the whole tree is +/// re-printed. The match is on the parsed AST subtree, not a substring of the +/// text, so a reducer text that is a textual prefix of a *different* reducer +/// subexpression (`sum(p[*])` vs `sum(p[*] + 1)`) is never falsely matched. +/// +/// Returns `Err([`PartialEquationError`])` when `equation_text` does not parse +/// (a genuine parse error, or an empty/whitespace equation that yields no AST) +/// *and there are reducers to substitute*: with no AST there is no reducer +/// subexpression to replace, so returning the input unchanged would let the +/// inline reducer survive into the `agg → target` partial -- a partial that +/// references the live reducer instead of the hoisted aggregate node, a +/// wrong-but-clean-compiling link score (the agg-substitution-omission sibling +/// of the GH #311 PREVIOUS-omission hazard; GH #661). The db-bearing caller +/// converts the error into a `Warning` (via `emit_ltm_partial_equation_warning`) +/// and skips the variable. The failure is effectively unreachable in production +/// (the input is a `print_eqn` re-print of an already-parsed AST), so this is +/// defense-in-depth. +/// +/// The empty-`reducers` case is a pure pass-through that never parses (there +/// is nothing to substitute), so it returns `Ok` with the text unchanged even +/// for otherwise-unparseable input. +/// +/// Test-only since Track A stage 1: the agg-substitution is now a POST-transform +/// lowering that runs on the wrapped AST via [`substitute_reducers_in_expr0`] +/// (the generators call that directly), so the text-level wrapper survives only +/// as the unit-tested entry point for the substitution behavior. +#[cfg(test)] +pub(crate) fn substitute_reducers_in_equation( + equation_text: &str, + reducers: &HashMap, +) -> Result { + if reducers.is_empty() { + return Ok(equation_text.to_string()); + } + let Ok(Some(ast)) = Expr0::new(equation_text, LexerType::Equation) else { + return Err(PartialEquationError::new(equation_text)); + }; + Ok(print_eqn(&substitute_reducers_in_expr0(ast, reducers))) +} + +/// The agg-substitution POST-transform lowering (Track A stage 1): rewrite each +/// hoisted reducer subexpression in an already-wrapped `Expr0` to its synthetic +/// agg name in place, keyed by canonical `print_eqn` text (`reducers` maps +/// reducer text -> agg name). It runs on the WRAPPED AST -- AFTER the +/// ceteris-paribus wrap held one hoisted reducer live and froze the rest whole +/// -- so the held-live reducer becomes a bare agg name and each frozen +/// `PREVIOUS(SUM(..))` becomes `PREVIOUS(agg)`, byte-for-byte matching the old +/// wrap-of-agg-substituted-text composition. It deliberately does NOT +/// re-classify (a pure text match), so stage 2 can retarget the wrap onto the +/// occurrence IR without touching this lowering. The `agg → target` generators +/// call it directly; the text-level `substitute_reducers_in_equation` wrapper +/// is now test-only. +pub(super) fn substitute_reducers_in_expr0( + expr: Expr0, + reducers: &HashMap, +) -> Expr0 { + // A whole-subtree match wins before descending: a reducer App is opaque + // -- once it matches an agg, we don't recurse into its (now-irrelevant) + // argument. + if let Some(agg_name) = reducers.get(&print_eqn(&expr)) { + return Expr0::Var( + crate::common::RawIdent::new_from_str(agg_name), + crate::ast::Loc::default(), + ); + } + match expr { + Expr0::Const(..) | Expr0::Var(..) => expr, + Expr0::Subscript(ident, indices, loc) => { + // A reducer can appear as (or inside) a subscript index expression + // -- `stock[SUM(idx[*])]` -- and `walk_subexpr_for_aggs` hoists it + // into a synthetic agg by descending into `IndexExpr2::Expr` / + // `IndexExpr2::Range`, so the substituter must mirror that descent. + // Wildcard / star-range / `@N` indices carry no `Expr0`, so they + // pass through unchanged. + let indices = indices + .into_iter() + .map(|idx| match idx { + IndexExpr0::Expr(e) => { + IndexExpr0::Expr(substitute_reducers_in_expr0(e, reducers)) + } + IndexExpr0::Range(l, r, loc) => IndexExpr0::Range( + substitute_reducers_in_expr0(l, reducers), + substitute_reducers_in_expr0(r, reducers), + loc, + ), + IndexExpr0::Wildcard(_) + | IndexExpr0::StarRange(_, _) + | IndexExpr0::DimPosition(_, _) => idx, + }) + .collect(); + Expr0::Subscript(ident, indices, loc) + } + Expr0::App(UntypedBuiltinFn(name, args), loc) => { + let args = args + .into_iter() + .map(|a| substitute_reducers_in_expr0(a, reducers)) + .collect(); + Expr0::App(UntypedBuiltinFn(name, args), loc) + } + Expr0::Op1(op, inner, loc) => Expr0::Op1( + op, + Box::new(substitute_reducers_in_expr0(*inner, reducers)), + loc, + ), + Expr0::Op2(op, lhs, rhs, loc) => Expr0::Op2( + op, + Box::new(substitute_reducers_in_expr0(*lhs, reducers)), + Box::new(substitute_reducers_in_expr0(*rhs, reducers)), + loc, + ), + Expr0::If(cond, then_e, else_e, loc) => Expr0::If( + Box::new(substitute_reducers_in_expr0(*cond, reducers)), + Box::new(substitute_reducers_in_expr0(*then_e, reducers)), + Box::new(substitute_reducers_in_expr0(*else_e, reducers)), + loc, + ), + } +} From 6993b9b6710915ba760321f4ebdfb36ddbaa17f5 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sat, 18 Jul 2026 17:26:16 -0700 Subject: [PATCH 04/14] engine: single-source the LTM other-dep verdict; prove occurrence alignment Groundwork for retiring the Expr0 mirror classifier family. Two pieces, both mechanism-verified: The other-dep iterated-subscript verdict (Collapse / Mismatch / NotIterated, which decides whether a non-live dep's subscript collapses to a bare PREVIOUS(dep), dooms the changed-first partial, or wraps normally) now has one implementation: the transform delegates to db::ltm_ir::derive_other_dep_verdict, derived from the occurrence IR's per-axis vocabulary. The delegation is arm-by-arm equivalent to the deleted duplicate, including the arity-dominance corner where the two families labeled an over-declared-arity axis differently: the arity guard returns Mismatch before any per-axis arm is consulted, so the labeling difference was unobservable (pinned by verdict_ignores_over_arity_axis_labeling). The classifier-agreement gate additionally asserts, for every fixture, that the occurrence IR's per-occurrence stream aligns one-to-one with the wrap walk's reference visits on the printed-and-reparsed text -- the premise a SiteId-keyed selection bridge depends on -- with new fixtures covering already-lagged, index-nested, LOOKUP, reducer, and module-output shapes. Misalignment fails with a diagnostic naming the path, turning the printer/parser-asymmetry class loud in the gate before any production switch. Two Fig. 2 selection semantics (index-nested exclusion from live capture, PREVIOUS/INIT verbatim-return) gain direct mutation-verified pins. The wrap itself still decides shapes on reparsed Expr0; the follow-on flips those decisions onto the IR and deletes the remaining classifiers. The GH #971 identifier_set fallback now marks itself loudly if it ever fires (corpus-unreachable today), so its planned deletion can proceed with confidence. --- src/simlin-engine/src/db.rs | 35 +++- .../already_lagged_other_dep.txt | 2 + .../reducer_index_nested_freeze.txt | 2 + src/simlin-engine/src/db/ltm_char_tests.rs | 158 ++++++++++++++ src/simlin-engine/src/db/ltm_ir.rs | 102 +++++++++ src/simlin-engine/src/db/ltm_ir_tests.rs | 148 ++++++------- src/simlin-engine/src/ltm_augment.rs | 143 +++++++------ src/simlin-engine/src/ltm_augment_tests.rs | 77 +++++++ .../src/ltm_classifier_agreement_tests.rs | 194 +++++++++++++++++- 9 files changed, 710 insertions(+), 151 deletions(-) create mode 100644 src/simlin-engine/src/db/ltm_char_golden/already_lagged_other_dep.txt create mode 100644 src/simlin-engine/src/db/ltm_char_golden/reducer_index_nested_freeze.txt diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index 762512702..9b5cf419f 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -855,7 +855,18 @@ pub(crate) fn module_link_score_equation( db, model, project, from_name, to_name, ) .or_else(|| { - to_var + // GH #971 / Track A stage 3 prep: the deterministic IR pick above + // should name the live output for every module->variable edge that + // receives a real partial (the occurrence IR enumerates every + // module-output composite `to` reads, in document order). The + // per-process-random `identifier_set` scan survives ONLY as a + // defensive fallback for the (expected-unreachable) case where the + // IR recorded no `ModuleOutput` occurrence for `to`. Mark LOUDLY + // whenever the scan actually RESCUES (IR missed, scan found one): + // stage 3 needs a fired-or-not signal before it can delete the scan + // with confidence. The rescued ref itself is unchanged -- this only + // warns; it never alters the emitted score. + let rescued = to_var .ast() .map(|ast| crate::variable::identifier_set(ast, &[], None)) .and_then(|deps| { @@ -863,7 +874,27 @@ pub(crate) fn module_link_score_equation( deps.into_iter() .find(|d| d.as_str().starts_with(&prefix)) .map(|d| d.to_string()) - }) + }); + if rescued.is_some() { + // In test/debug builds the assertion is the loud marker (the + // scan is meant to be dead, so a fired assert is real signal a + // module-output occurrence is missing from the IR). In release + // builds `debug_assert!` is a no-op, so also emit a warning line + // -- the repo's `eprintln!` idiom for an unexpected internal + // condition (cf. `model.rs`, `dimensions.rs`). + debug_assert!( + false, + "GH #971: module-output IR pick missed `{from_name}\u{00B7}` \ + in `{to_name}`; the identifier_set fallback rescued it. The \ + occurrence IR should enumerate every module-output composite \ + (Track A stage 3 deletes this fallback once it is proven dead)." + ); + eprintln!( + "warning: LTM module-output IR pick missed `{from_name}\u{00B7}` \ + in `{to_name}`; used the identifier_set fallback (GH #971)." + ); + } + rescued }); match from_output { Some(output_ref) => { diff --git a/src/simlin-engine/src/db/ltm_char_golden/already_lagged_other_dep.txt b/src/simlin-engine/src/db/ltm_char_golden/already_lagged_other_dep.txt new file mode 100644 index 000000000..bca137f8d --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/already_lagged_other_dep.txt @@ -0,0 +1,2 @@ +$⁚ltm⁚link_score⁚from→to +scalar: if (TIME = INITIAL_TIME) then 0 else if ((to - PREVIOUS(to)) = 0) OR ((from - PREVIOUS(from)) = 0) then 0 else SAFEDIV(((from + previous(g, 0)) - PREVIOUS(to)), ABS((to - PREVIOUS(to))), 0) * SIGN((from - PREVIOUS(from))) diff --git a/src/simlin-engine/src/db/ltm_char_golden/reducer_index_nested_freeze.txt b/src/simlin-engine/src/db/ltm_char_golden/reducer_index_nested_freeze.txt new file mode 100644 index 000000000..53ec11ca0 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/reducer_index_nested_freeze.txt @@ -0,0 +1,2 @@ +$⁚ltm⁚link_score⁚from→to +scalar: if (TIME = INITIAL_TIME) then 0 else if ((to - PREVIOUS(to)) = 0) OR ((from - PREVIOUS(from)) = 0) then 0 else SAFEDIV(((PREVIOUS(sum(w[from, *])) + from) - PREVIOUS(to)), ABS((to - PREVIOUS(to))), 0) * SIGN((from - PREVIOUS(from))) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index f99e713c3..aec08ace0 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -1118,3 +1118,161 @@ fn char_arrayed_target_slot_scores() { let actual = dump_synthetic_vars(arrayed_target_model(), true, "link_score\u{205A}pop"); assert_golden("arrayed_target_slot_scores", &actual); } + +// --------------------------------------------------------------------------- +// Model F (Track A3 stage 2, review finding 2): the GH #517 whole-reducer +// freeze over an INDEX-NESTED live-source occurrence (Fig. 2 Q4). +// +// `to = SUM(w[from, *]) + from`, edge `(from -> to)`, `Bare` shape. `from` +// (the live source) occurs TWICE in `to`'s equation: +// * bare, outside any reducer (`+ from`) -- the live occurrence; +// * index-nested, inside the reducer's subscript (`w[from, *]`). +// +// The changed-first partial must FREEZE THE WHOLE REDUCER -- +// `PREVIOUS(SUM(w[from, *])) + from` -- because `expr0_contains_live_match`'s +// Subscript arm matches only a subscript whose HEAD is the live source, never +// an index-nested occurrence (`ltm_augment.rs`). Recursing into the reducer +// instead would emit `SUM(PREVIOUS(w[from, *]))` (a PREVIOUS of an array view, +// which has no LoadPrev path -- a loud compile failure and a silently-zeroed +// score, GH #517). +// +// This pins the exact Fig. 2 Q4 selection semantics the stage-2 occurrence +// switch must reproduce via `occ.index_nested`: an index-nested occurrence is +// excluded from the reducer-containment test AND from live selection. Before +// this golden, mutating `expr0_contains_live_match` to count index-nested +// occurrences (the exact stage-2 `occ.index_nested` mishandling) passed the +// entire corpus green while changing this text -- verified: the mutation flips +// this golden's numerator from the changed-first whole-reducer freeze +// `PREVIOUS(sum(w[from, *])) + from` to the changed-LAST form +// `to - (sum(w[from, *]) + PREVIOUS(from))` (the reducer held live, the bare +// `from` frozen). Both COMPILE and are non-zero, so only this exact-text pin +// catches the drift -- a runtime compile/non-zero guard would not. The +// scalar-element form (`SUM(w[from])`) is pinned directly at the wrap level by +// `partial_freezes_whole_reducer_over_index_nested_live_source` in +// `ltm_augment_tests.rs`; the runtime guard below is a separate silent-zero +// backstop that this shape's score stays materially live. +// --------------------------------------------------------------------------- + +fn reducer_index_nested_model() -> datamodel::Project { + TestProject::new("reducer_index_nested_char") + .named_dimension("Slot", &["s1", "s2"]) + .named_dimension("K", &["k1", "k2"]) + .array_aux("w[Slot,K]", "0.5") + .aux("from", "1", None) + .aux("to", "SUM(w[from, *]) + from", None) + .stock("acc", "0", &["toflow"], &[], None) + .flow("toflow", "to", None) + .build_datamodel() +} + +#[test] +fn char_reducer_index_nested_freeze() { + let actual = dump_synthetic_vars( + reducer_index_nested_model(), + true, + "link_score\u{205A}from\u{2192}to", + ); + assert_golden("reducer_index_nested_freeze", &actual); +} + +// Finding 2 materiality guard for the index-nested reducer-freeze shape, +// mirroring the `agg_nested_reducer_preserves_loud_failure_not_silent_zero` +// idiom: the byte-parity behavior of `to = SUM(w[from, *]) + from` is a LOUD +// failure, not a silent compile. Because `from` is a dynamic index, the frozen +// `PREVIOUS(sum(w[from, *]))` desugars to a synthesized scalar PREVIOUS-helper +// over a dynamic-index reducer, which the engine cannot compile (a pre-existing +// dynamic-index-reducer limitation) and reports as a `Warning`. +// +// NOTE on scope: the specific index-nested-exclusion drift (recursing into the +// reducer to hold the index-nested `from` live) is a TEXT-only change at +// runtime -- verified: it flips the wrap to the changed-last form +// `to - (sum(w[from, *]) + PREVIOUS(from))`, which STILL fails to compile (a +// dynamic-index reducer inline in a scalar guard equation), so both baseline +// and mutation loud-fail. That drift is caught by the `reducer_index_nested_freeze` +// text golden and the wrap unit test, not here. This guard is the complementary +// silent-vs-loud backstop: it pins that this Fig. 2 Q4 shape LOUDLY fails rather +// than silently compiling to a wrong-valued score -- the codebase's preferred +// degradation (GH #311/#661/#743). A future selector that made this shape +// silently compile (dropping the warning) turns this red. + +fn reducer_index_nested_feedback_model() -> datamodel::Project { + TestProject::new("reducer_index_nested_feedback") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Slot", &["s1", "s2"]) + .named_dimension("K", &["k1", "k2"]) + .array_aux("w[Slot,K]", "0.5") + .aux("from", "1 + (TIME MOD 2)", None) + .aux("to", "SUM(w[from, *]) + from", None) + .stock("acc", "0", &["toflow"], &[], None) + .flow("toflow", "to", None) + .build_datamodel() +} + +#[test] +fn reducer_index_nested_freeze_preserves_loud_failure_not_silent_compile() { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + use salsa::Setter; + + let project = reducer_index_nested_feedback_model(); + let mut db = SimlinDb::default(); + let (source_project, source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + + let diags = collect_model_diagnostics(&db, source_model, source_project); + let from_to_failures: Vec<_> = diags + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .filter_map(|d| d.variable.clone()) + .filter(|v| v.contains("link_score\u{205A}from\u{2192}to")) + .collect(); + assert!( + !from_to_failures.is_empty(), + "the changed-first whole-reducer freeze over a dynamic index-nested live \ + source must preserve HEAD's loud compile-failure warning (not silently \ + compile the changed-last form that holds the reducer live); a missing \ + warning means the index-nested occurrence was held live" + ); +} + +// --------------------------------------------------------------------------- +// Model G (Track A3 stage 2, review finding 2): an already-lagged other-dep +// (Fig. 2 Q3). +// +// `to = from + PREVIOUS(g)`, edge `(from -> to)`, `Bare` shape. `g` occurs +// only inside `PREVIOUS(g)` -- it is already lagged. The changed-first partial +// holds `from` live and must LEAVE `PREVIOUS(g)` untouched +// (`from + PREVIOUS(g)`), NOT re-wrap it to `PREVIOUS(PREVIOUS(g))` (a t-2 +// read). This pins the `already_lagged` selection semantics the stage-2 switch +// must reproduce via `occ.already_lagged`: it suppresses the wrap/freeze of an +// already-lagged occurrence but not its live selection. +// --------------------------------------------------------------------------- + +fn already_lagged_other_dep_model() -> datamodel::Project { + TestProject::new("already_lagged_char") + .aux("g", "3", None) + .aux("from", "1", None) + .aux("to", "from + PREVIOUS(g)", None) + .stock("acc", "0", &["toflow"], &[], None) + .flow("toflow", "to", None) + .build_datamodel() +} + +#[test] +fn char_already_lagged_other_dep() { + let actual = dump_synthetic_vars( + already_lagged_other_dep_model(), + true, + "link_score\u{205A}from\u{2192}to", + ); + assert_golden("already_lagged_other_dep", &actual); +} diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index 1e2246412..d22172c2d 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -1060,6 +1060,108 @@ impl OccurrenceAxis { } } +/// The `Collapse` / `Mismatch` / `NotIterated` verdict for an iterated-dimension +/// subscript on a NON-live-source dependency (`pop[Region,Age]` in +/// `growth[Region,Age] = row_sum[Region] * c * pop[Region,Age]` while scoring +/// `(row_sum, growth)`). See [`derive_other_dep_verdict`] and [`OccurrenceAxis`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum OtherDepVerdict { + /// Collapse the subscript to a bare `Var(dep)` before the `PREVIOUS()` wrap + /// (the indices correspond position-for-position to the dep's declared axes, + /// or the dep's dims are un-threadable and the historical permissive + /// collapse applies). + Collapse, + /// Not an iterated-dim subscript at all (a literal, wildcard, dynamic index, + /// a name outside the target's iterated dims, or more indices than the + /// target has iterated dims): normal subscript handling. + NotIterated, + /// Every index names a target-iterated dimension, but the dep's declared + /// axes provably do NOT correspond positionally (transposed / arity- + /// mismatched / mapped without a usable positional correspondence). + /// Collapsing would freeze the WRONG element (GH #526). + Mismatch, +} + +/// The single source of truth for the other-dep iterated-subscript verdict, +/// derived from a subscript occurrence's per-axis [`OccurrenceAxis`] +/// classification plus the two arity facts the occurrence does not itself carry +/// -- the dep's declared arity `dep_arity` (`None` = un-threadable: the dep is +/// absent from the variable map / has no declared dims) and the target's +/// iterated-dim count `target_iterated_count`. +/// +/// `ltm_augment::classify_other_dep_iterated_dim_subscript` (the Expr0-side +/// classifier the ceteris-paribus wrap consults) builds `axes` from the parsed +/// subscript and delegates here. That wrap-side axis construction is still a +/// textual mirror of the IR walker's [`classify_occurrence_axes`] -- Track A3 +/// stage 2 has not yet retargeted the wrap onto the occurrence stream -- but the +/// two derivations provably cannot produce a different VERDICT, the only value +/// the wrap consumes: +/// +/// - Every IN-arity position (index `< dep_arity`) agrees: both derivations gate +/// the iterated-dim lineup on the identical [`crate::ltm_agg::iterated_axis_slot_elements`] +/// (compare [`crate::ltm_agg::classify_axis_access`]'s `Var` arm with +/// `ltm_augment::other_dep_axis_lines_up`), so they label each such index the +/// same `Iterated` / `MismatchedIterated`. +/// - The two disagree ONLY on an OVER-declared-arity index (index `>= dep_arity`): +/// the mirror's `dep_dims.get(i) == None` arm labels it `Iterated{d,d}`, the +/// IR's `source_dims.get(i) == None` arm labels it `MismatchedIterated{d}`. +/// But such a position exists only when `axes.len() > dep_arity`, and the +/// arity check below returns `Mismatch` for that whole case BEFORE inspecting +/// the per-axis arms. +/// +/// So the arity guard DOMINATES the sole labeling difference: the silent-zero +/// drift the two-family "must stay in sync" contract guarded against is +/// structurally impossible for this verdict, whichever family built `axes`. +/// Stage 2 will still delete the wrap-side axis construction (collapsing to one +/// classifier family), but the verdict itself is already single-sourced here. +/// See [`OccurrenceAxis`]'s rustdoc for the full rule; the two load-bearing +/// arity corners (under-arity all-`Iterated` => `Mismatch`; over-target-arity => +/// `NotIterated`) are pinned by `under_arity_iterated_subscript_is_mismatch_not_collapse` +/// / `over_target_arity_iterated_subscript_is_not_iterated`, and the +/// dominance argument by `verdict_ignores_over_arity_axis_labeling`, in +/// `db::ltm_ir_tests`. +pub(crate) fn derive_other_dep_verdict( + axes: &[OccurrenceAxis], + dep_arity: Option, + target_iterated_count: usize, +) -> OtherDepVerdict { + // Precondition (else normal subscript handling): a non-empty subscript, no + // more indices than the target has iterated dims, and every index a bare + // target-iterated-dim name -- i.e. every axis `Iterated` or + // `MismatchedIterated` (a `Pinned`/`Reduced`/`Dynamic` axis is a literal, + // wildcard, or dynamic index). + if axes.is_empty() || axes.len() > target_iterated_count { + return OtherDepVerdict::NotIterated; + } + let all_iterated_or_mismatched = axes.iter().all(|a| { + matches!( + a, + OccurrenceAxis::Iterated { .. } | OccurrenceAxis::MismatchedIterated { .. } + ) + }); + if !all_iterated_or_mismatched { + return OtherDepVerdict::NotIterated; + } + // The dep's declared arity gates the collapse: un-threadable keeps the + // transform's permissive collapse; a differing arity is a `Mismatch` + // (checked BEFORE the per-axis lineup so an over-declared-arity subscript + // whose trailing indices are `MismatchedIterated` from a missing source + // axis is a `Mismatch`, not mislabelled by the per-axis arms). + let Some(arity) = dep_arity else { + return OtherDepVerdict::Collapse; + }; + if axes.len() != arity { + return OtherDepVerdict::Mismatch; + } + if axes + .iter() + .any(|a| matches!(a, OccurrenceAxis::MismatchedIterated { .. })) + { + return OtherDepVerdict::Mismatch; + } + OtherDepVerdict::Collapse +} + /// How a per-occurrence reference routes, the occurrence-faithful counterpart /// of [`SiteRouting`]. Unlike `SiteRouting` (which the edge consumers split /// into one entry per routed agg), a single syntactic occurrence keeps ONE diff --git a/src/simlin-engine/src/db/ltm_ir_tests.rs b/src/simlin-engine/src/db/ltm_ir_tests.rs index b03632697..277b2a710 100644 --- a/src/simlin-engine/src/db/ltm_ir_tests.rs +++ b/src/simlin-engine/src/db/ltm_ir_tests.rs @@ -1375,83 +1375,22 @@ mod occurrence_ir_tests { }); } - // ── Other-dep verdict derivation (the A2b contract on `axes`) ─────────── + // ── Other-dep verdict derivation (the contract on `axes`) ────────────── // - // `OccurrenceAxis`'s rustdoc states the rule by which A2b will derive + // `OccurrenceAxis`'s rustdoc states the rule by which the transform derives // `ltm_augment::classify_other_dep_iterated_dim_subscript`'s - // `Collapse`/`Mismatch`/`NotIterated` verdict from an occurrence's `axes` - // (after deleting the Expr0 mirror classifier). The tests below are the - // executable form of that contract: [`derive_other_dep_verdict`] is the - // reference derivation, and the two corner tests pin the `axes` the walker - // actually produces for the arity shapes where a rule keyed on the per-axis - // arms ALONE (all-`Iterated` ⇒ Collapse, any-`MismatchedIterated` ⇒ - // Mismatch) diverges from the transform. Deriving the verdict requires the + // `Collapse`/`Mismatch`/`NotIterated` verdict from an occurrence's `axes`. + // Track A3 stage 2 promoted the reference derivation to the production + // `db::ltm_ir::derive_other_dep_verdict` -- the Expr0-side classifier now + // builds `axes` from the parsed subscript and DELEGATES to it, so the wrap + // and the IR cannot drift. The tests below exercise that promoted helper + // directly (via `use super::*`), and the two corner tests pin the `axes` the + // walker actually produces for the arity shapes where a rule keyed on the + // per-axis arms ALONE (all-`Iterated` ⇒ Collapse, any-`MismatchedIterated` + // ⇒ Mismatch) diverges from the transform. Deriving the verdict requires the // two arity facts the occurrence does not itself carry -- the dep's declared - // arity and the target's iterated-dim count -- both of which A2b holds. - - /// The three verdicts of `ltm_augment::OtherDepIteratedVerdict`, mirrored - /// here so the derivation contract is testable without reaching into - /// `ltm_augment`'s private types. - #[derive(Debug, PartialEq, Eq)] - enum DerivedVerdict { - Collapse, - Mismatch, - NotIterated, - } - - /// Reference implementation of the `OccurrenceAxis` rustdoc's derivation - /// rule -- the verdict A2b must compute for a subscript occurrence's - /// `axes`, given the dep's declared arity (`None` = un-threadable: the dep - /// is absent from the variable map / has no declared dims) and the target's - /// iterated-dim count. It reproduces - /// `classify_other_dep_iterated_dim_subscript` (`ltm_augment.rs`) exactly, - /// including the two arity guards a per-axis-only rule would miss: - /// - `axes.len() > target_iterated_count` ⇒ `NotIterated` (mirrors the - /// `indices.len() > target_iterated_dims.len()` gate, ltm_augment.rs:268); - /// - a threadable dep whose declared arity differs from `axes.len()` ⇒ - /// `Mismatch` (mirrors `index_dims.len() != dep_dims.len()`, - /// ltm_augment.rs:288) -- checked BEFORE the per-axis lineup so an - /// over-declared-arity subscript (whose trailing indices are - /// `MismatchedIterated` from a missing source axis) is a `Mismatch`, not - /// mislabelled by the per-axis arms. - fn derive_other_dep_verdict( - axes: &[OccurrenceAxis], - dep_arity: Option, - target_iterated_count: usize, - ) -> DerivedVerdict { - // Precondition (else normal subscript handling): a non-empty subscript, - // no more indices than the target has iterated dims, and every index a - // bare target-iterated-dim name -- i.e. every axis `Iterated` or - // `MismatchedIterated` (a `Pinned`/`Reduced`/`Dynamic` axis is a - // literal, wildcard, or dynamic index). - if axes.is_empty() || axes.len() > target_iterated_count { - return DerivedVerdict::NotIterated; - } - let all_iterated_or_mismatched = axes.iter().all(|a| { - matches!( - a, - OccurrenceAxis::Iterated { .. } | OccurrenceAxis::MismatchedIterated { .. } - ) - }); - if !all_iterated_or_mismatched { - return DerivedVerdict::NotIterated; - } - // The dep's declared arity gates the collapse: un-threadable keeps the - // transform's permissive collapse; a differing arity is a `Mismatch`. - let Some(arity) = dep_arity else { - return DerivedVerdict::Collapse; - }; - if axes.len() != arity { - return DerivedVerdict::Mismatch; - } - if axes - .iter() - .any(|a| matches!(a, OccurrenceAxis::MismatchedIterated { .. })) - { - return DerivedVerdict::Mismatch; - } - DerivedVerdict::Collapse - } + // arity and the target's iterated-dim count -- both of which the transform + // holds. fn iterated(dim: &str) -> OccurrenceAxis { OccurrenceAxis::Iterated { @@ -1472,27 +1411,27 @@ mod occurrence_ir_tests { // target [D1,D2]): all `Iterated`, arity matches ⇒ Collapse. assert_eq!( derive_other_dep_verdict(&[iterated("d1"), iterated("d2")], Some(2), 2), - DerivedVerdict::Collapse, + OtherDepVerdict::Collapse, ); // Transposed equal-arity (`arr[D2,D1]`): a `MismatchedIterated` axis // with matching arity ⇒ Mismatch (the GH #526 wrong-element freeze). assert_eq!( derive_other_dep_verdict(&[mismatched("d2"), mismatched("d1")], Some(2), 2), - DerivedVerdict::Mismatch, + OtherDepVerdict::Mismatch, ); // Under-arity (corner a): all `Iterated` but fewer indices than the // dep's declared arity ⇒ Mismatch, NOT the Collapse the all-`Iterated` // arms alone would give. assert_eq!( derive_other_dep_verdict(&[iterated("d1")], Some(2), 2), - DerivedVerdict::Mismatch, + OtherDepVerdict::Mismatch, ); // Over-target-arity (corner b): more indices than the target has // iterated dims ⇒ NotIterated, NOT the Mismatch the `MismatchedIterated` // arm alone would give. assert_eq!( derive_other_dep_verdict(&[iterated("d1"), mismatched("d1")], Some(2), 1), - DerivedVerdict::NotIterated, + OtherDepVerdict::NotIterated, ); // A non-iterated axis (a `Pinned` literal / `Dynamic` index) anywhere ⇒ // NotIterated: not an iterated-dim subscript at all. @@ -1502,17 +1441,17 @@ mod occurrence_ir_tests { Some(2), 2, ), - DerivedVerdict::NotIterated, + OtherDepVerdict::NotIterated, ); assert_eq!( derive_other_dep_verdict(&[OccurrenceAxis::Dynamic], Some(1), 1), - DerivedVerdict::NotIterated, + OtherDepVerdict::NotIterated, ); // Un-threadable dep (declared dims unknown) keeps the permissive // collapse regardless of the per-axis arms. assert_eq!( derive_other_dep_verdict(&[iterated("d1"), mismatched("d2")], None, 2), - DerivedVerdict::Collapse, + OtherDepVerdict::Collapse, ); } @@ -1552,7 +1491,7 @@ mod occurrence_ir_tests { // dep arity 2, target iterated-dim count 2. assert_eq!( derive_other_dep_verdict(&arr[0].axes, Some(2), 2), - DerivedVerdict::Mismatch, + OtherDepVerdict::Mismatch, "under-arity must derive Mismatch, not Collapse" ); }); @@ -1588,9 +1527,52 @@ mod occurrence_ir_tests { // dep arity 2, target iterated-dim count 1. assert_eq!( derive_other_dep_verdict(&arr[0].axes, Some(2), 1), - DerivedVerdict::NotIterated, + OtherDepVerdict::NotIterated, "an over-target-arity subscript must derive NotIterated, not Mismatch" ); }); } + + #[test] + fn verdict_ignores_over_arity_axis_labeling() { + // Track A3 stage 2 / finding-3 verdict-equivalence. The Expr0-side + // mirror (`ltm_augment::classify_other_dep_iterated_dim_subscript`) and + // the IR walker (`classify_occurrence_axes`) can LABEL a per-axis index + // differently at exactly ONE kind of position: an index that overflows + // the dep's declared arity. The mirror's `dep_dims.get(i) == None` arm + // marks it `Iterated{d,d}`; the IR's `source_dims.get(i) == None` arm + // marks the SAME over-arity index `MismatchedIterated{d}`. Every + // in-arity position agrees (both derivations gate the lineup on the + // identical `ltm_agg::iterated_axis_slot_elements`; see + // `classify_axis_access` vs `other_dep_axis_lines_up`). + // + // But an over-declared-arity position exists only when `axes.len() > + // dep_arity`, and the arity check in `derive_other_dep_verdict` returns + // `Mismatch` for that whole case BEFORE inspecting the per-axis arms. So + // the sole labeling difference is dominated: the verdict cannot differ + // between the two derivations. That is what makes the shared verdict + // rule single-sourced -- the "silent drift" the two-family "must stay in + // sync" contract guards against is structurally impossible for the + // VERDICT (the only value the wrap consumes), whichever family built the + // `axes`. + // + // 3 indices, dep declared arity 2 (index 2 overflows), target + // iterated-dim count 3. + let mirror = [iterated("d1"), iterated("d2"), iterated("d3")]; + let ir = [iterated("d1"), iterated("d2"), mismatched("d3")]; + assert_ne!( + mirror[2], ir[2], + "the two families label the over-arity index differently" + ); + assert_eq!( + derive_other_dep_verdict(&mirror, Some(2), 3), + derive_other_dep_verdict(&ir, Some(2), 3), + "the arity guard dominates the labeling difference: same verdict either way" + ); + assert_eq!( + derive_other_dep_verdict(&ir, Some(2), 3), + OtherDepVerdict::Mismatch, + "axes.len() (3) != dep arity (2) => Mismatch" + ); + } } diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 1019d938e..68475f9a4 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -20,6 +20,7 @@ use crate::variable::{Variable, identifier_set}; use std::collections::{HashMap, HashSet}; use crate::db::RefShape; +use crate::db::ltm_ir::{OccurrenceAxis, OtherDepVerdict, derive_other_dep_verdict}; /// The implicit WITH-LOOKUP rules (GH #910), in their own file only to keep this /// one under the project line-count lint. Re-exported below so callers keep @@ -212,35 +213,10 @@ fn classify_expr0_per_element_axes( Some(axes) } -/// Verdict of [`classify_other_dep_iterated_dim_subscript`] for an -/// iterated-dimension subscript on a non-live-source dependency. -#[derive(Debug, PartialEq, Eq)] -enum OtherDepIteratedVerdict { - /// Collapse the subscript to a bare `Var(dep)` before the `PREVIOUS()` - /// wrap: the indices correspond position-for-position to the dep's - /// declared axes (so the bare reference reads the same element under - /// the target's A2A expansion), or the dep's dims are un-threadable - /// and the historical permissive collapse applies. - Collapse, - /// Not an iterated-dim subscript at all (a literal, wildcard, or - /// dynamic index, or a name outside the target's iterated dims): - /// normal subscript handling. - NotIterated, - /// Every index names a target-iterated dimension, but the dep's known - /// declared axes provably do NOT correspond positionally -- a - /// transposed (`arr[D2,D1]` for `arr[D1,D2]`) or - /// position-mismatched-mapped reference. Collapsing would freeze the - /// WRONG element (a silent magnitude error, GH #526); the caller - /// abandons the changed-first partial instead (the changed-last - /// fallback keeps the dep live and verbatim, or the partial fails - /// loudly). - Mismatch, -} - /// Classify an iterated-dimension subscript on a *non-live-source* /// dependency (e.g. `pop[Region,Age]` in `growth[Region,Age] = /// row_sum[Region] * c * pop[Region,Age]` while building the partial for -/// `(row_sum, growth)`). On [`OtherDepIteratedVerdict::Collapse`], +/// `(row_sum, growth)`). On [`OtherDepVerdict::Collapse`], /// `wrap_non_matching_in_previous` collapses the subscript to a bare /// `Var(dep)` before wrapping it in `PREVIOUS()` -- avoiding the /// `PREVIOUS(Subscript(...))` codegen assertion. @@ -248,7 +224,7 @@ enum OtherDepIteratedVerdict { /// The firing precondition is unchanged from the pre-GH#526 recognizer: /// every index is a bare `Var` naming one of the target's iterated /// dimensions, with at most as many indices as the target has iterated -/// dimensions (anything else is [`OtherDepIteratedVerdict::NotIterated`]). +/// dimensions (anything else is [`OtherDepVerdict::NotIterated`]). /// /// What the verdict adds (GH #526) is the position-and-mapping /// correspondence check against the dep's DECLARED dimensions @@ -275,43 +251,70 @@ enum OtherDepIteratedVerdict { /// source->agg per-row partials in `generate_nonlinear_body_partial`; /// this is the other-dep iterated-subscript collapse in target-equation /// partials.) +/// +/// Track A3 stage 2 status: the verdict RULE is single-sourced on +/// [`derive_other_dep_verdict`] (`db::ltm_ir`) -- the same rule the occurrence +/// IR's `axes` feed -- so the verdict cannot drift whichever family builds +/// `axes` (that rustdoc proves the shared arity guard dominates the sole +/// per-axis labeling difference; `verdict_ignores_over_arity_axis_labeling` +/// pins it). What is NOT yet single-sourced is the per-axis [`OccurrenceAxis`] +/// construction below -- still a textual mirror of `classify_occurrence_axes`; +/// retiring it needs the SiteId occurrence bridge into the wrap (deferred +/// stage-2 work; see the stage-2 report for why the wrap-surface + per-element +/// lowering conversion is multi-round). fn classify_other_dep_iterated_dim_subscript( dep: &Ident, indices: &[IndexExpr0], ctx: Option<&IteratedDimCtx<'_>>, -) -> OtherDepIteratedVerdict { +) -> OtherDepVerdict { let Some(ctx) = ctx else { - return OtherDepIteratedVerdict::NotIterated; + return OtherDepVerdict::NotIterated; }; + // Short-circuit before touching `dep_dims`: a non-empty subscript with no + // more indices than the target has iterated dims. (`derive_other_dep_verdict` + // re-checks these, but building the axes below already assumes them.) if indices.is_empty() || indices.len() > ctx.target_iterated_dims.len() { - return OtherDepIteratedVerdict::NotIterated; + return OtherDepVerdict::NotIterated; } - let mut index_dims: Vec = Vec::with_capacity(indices.len()); - for idx in indices { - match idx { - IndexExpr0::Expr(Expr0::Var(name, _)) => { - let d = canonicalize(name.as_str()).into_owned(); - if !ctx.target_iterated_dims.iter().any(|t| t == &d) { - return OtherDepIteratedVerdict::NotIterated; + let dep_dims = ctx.dep_dims.and_then(|m| m.get(dep.as_str())); + // Build the per-axis classification the promoted verdict rule consumes: an + // index lined up (by name or positional mapping) with the dep's declared + // axis at that position is `Iterated`, else `MismatchedIterated`. An index + // that is NOT a bare target-iterated-dim `Var` makes the whole subscript + // `NotIterated`. When the dep is un-threadable (`dep_dims` is `None`) the + // verdict's `None`-arity arm collapses regardless of the per-axis marking, + // and a position with no declared dep axis (an over-dep-arity index) is + // caught by the verdict's arity check first -- so marking those `Iterated` + // is sound (it only needs to keep every axis iterated/mismatched). + let mut axes: Vec = Vec::with_capacity(indices.len()); + for (i, idx) in indices.iter().enumerate() { + let IndexExpr0::Expr(Expr0::Var(name, _)) = idx else { + return OtherDepVerdict::NotIterated; + }; + let d = canonicalize(name.as_str()).into_owned(); + if !ctx.target_iterated_dims.iter().any(|t| t == &d) { + return OtherDepVerdict::NotIterated; + } + let axis = match dep_dims.and_then(|dd| dd.get(i)) { + Some(dep_dim) if other_dep_axis_lines_up(&d, dep_dim, ctx) => { + OccurrenceAxis::Iterated { + dim: d, + source_dim: canonicalize(dep_dim.name()).into_owned(), } - index_dims.push(d); } - _ => return OtherDepIteratedVerdict::NotIterated, - } - } - let Some(dep_dims) = ctx.dep_dims.and_then(|m| m.get(dep.as_str())) else { - // Un-threadable dep dims: the historical permissive collapse. - return OtherDepIteratedVerdict::Collapse; - }; - if index_dims.len() != dep_dims.len() { - return OtherDepIteratedVerdict::Mismatch; - } - for (d, dep_dim) in index_dims.iter().zip(dep_dims) { - if !other_dep_axis_lines_up(d, dep_dim, ctx) { - return OtherDepIteratedVerdict::Mismatch; - } + Some(_) => OccurrenceAxis::MismatchedIterated { dim: d }, + None => OccurrenceAxis::Iterated { + dim: d.clone(), + source_dim: d, + }, + }; + axes.push(axis); } - OtherDepIteratedVerdict::Collapse + derive_other_dep_verdict( + &axes, + dep_dims.map(|dd| dd.len()), + ctx.target_iterated_dims.len(), + ) } /// Does iterated-dimension index `d` (canonical) line up with a non-live @@ -381,14 +384,24 @@ fn is_literal_element_index( /// Resolve a single subscript index to a literal element name, mirroring /// `db::ltm_ir::resolve_literal_index` (the Expr2 sibling) so both -/// classifiers agree on what counts as a "literal element". The two -/// must stay in sync: the edge emitter uses the Expr2 classifier and -/// the partial-equation builder uses this Expr0 sibling -- if they -/// disagree (for example on out-of-range integer literals), the edge -/// emitter classifies as `DynamicIndex` while the partial builder -/// classifies as `FixedIndex(...)`, the shape comparison in -/// `wrap_non_matching_in_previous` fails, and the live reference is -/// wrapped in `PREVIOUS()` -- silently zeroing the link score. +/// classifiers agree on what counts as a "literal element". +/// +/// Two-family invariant (the tracked Track A3 stage-2/3 surface): the edge +/// emitter classifies each reference on the `Expr2` AST (`db::ltm_ir`), while +/// the ceteris-paribus wrap re-derives the shape on the printed-and-reparsed +/// `Expr0` via this sibling. If they disagree (e.g. on an out-of-range integer +/// literal -- `Expr2` -> `DynamicIndex`, `Expr0` -> `FixedIndex(...)`), the +/// shape comparison in `wrap_non_matching_in_previous` fails, every occurrence +/// of the source is `PREVIOUS`-wrapped, and the link score silently zeroes. +/// The A1 differential gate (`ltm_classifier_agreement_tests`) turns any drift +/// on a corpus-covered shape into a LOUD test failure; what it cannot cover is +/// a novel PRODUCTION shape, which is exactly why stage 2 deletes this Expr0 +/// sibling -- routing the shape/literal decision through the `db::ltm_ir` +/// occurrence stream (the SiteId bridge) makes the drift structurally +/// impossible for EVERY shape, not just the covered ones. (The other-dep +/// VERDICT is already single-sourced on `db::ltm_ir::derive_other_dep_verdict`; +/// the residual per-element lowering blocker is documented in the stage-2 +/// report.) /// /// Element names appear as `Var` nodes; integer literals appear as /// `Const` nodes whose text is the integer. Either form is validated @@ -638,7 +651,7 @@ struct WrapOutcome { /// [`wrap_non_matching_in_previous`]. live_ref: Option, /// GH #526: set when a KNOWN position-mismatched other-dep iterated - /// subscript was encountered ([`OtherDepIteratedVerdict::Mismatch`]). + /// subscript was encountered ([`OtherDepVerdict::Mismatch`]). /// The changed-first partial is then unusable -- collapsing would /// freeze the wrong element, and not collapsing leaves a /// `PREVIOUS(Subscript(dim-name indices))` whose capture helper cannot @@ -848,10 +861,10 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO // fire on this path anyway (its `dep_dims` are `None`), so // skipping the whole verdict preserves behavior. match classify_other_dep_iterated_dim_subscript(&canonical, &indices, iter_ctx) { - OtherDepIteratedVerdict::Collapse => { + OtherDepVerdict::Collapse => { return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out); } - OtherDepIteratedVerdict::Mismatch => { + OtherDepVerdict::Mismatch => { // GH #526: collapsing would freeze the WRONG element. // Record the doom and fall through to the normal // wrap; the caller abandons this changed-first form @@ -860,7 +873,7 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO // emitted. out.other_dep_mismatch = true; } - OtherDepIteratedVerdict::NotIterated => {} + OtherDepVerdict::NotIterated => {} } } // Classify the subscript's shape using the ORIGINAL indices diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index 883ab9584..a01a8853b 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -2885,6 +2885,83 @@ fn partial_equation_does_not_rewrap_inside_previous() { ); } +/// GH #517 / Fig. 2 Q4 (Track A3 stage 2, review finding 2): an INDEX-NESTED +/// occurrence of the live source must be EXCLUDED from the reducer-containment +/// test, so a reducer whose only live-source occurrence is index-nested freezes +/// WHOLE rather than recursing. +/// +/// `to = SUM(w[from]) + from` with live source `from` (Bare): `from` occurs +/// bare (outside the reducer -- the live occurrence) AND index-nested inside +/// `w[from]`. `expr0_contains_live_match`'s Subscript arm matches only a +/// subscript whose HEAD is the live source, so the index-nested `from` never +/// makes the reducer "hold the live reference": the whole reducer is frozen +/// (`PREVIOUS(sum(w[from]))`) and only the bare `from` stays live. +/// +/// This is the exact silent-drift the stage-2 occurrence switch must reproduce +/// via `occ.index_nested`. The scalar-element `SUM(w[from])` form (as opposed +/// to the array-slice `SUM(w[from, *])` production golden) is the one that +/// compiles under BOTH the correct and the mutated behavior -- so nothing but +/// this exact-text pin catches a selector that mishandles `index_nested` here: +/// recursing would emit `sum(PREVIOUS(w[from])) + from` (the index-nested +/// `from` held live), a compiling but value-divergent partial. +#[test] +fn partial_freezes_whole_reducer_over_index_nested_live_source() { + let deps = deps_set(&["w", "from"]); + let live = Ident::::new("from"); + let shape = RefShape::Bare; + + let partial = + build_partial_equation_shaped("SUM(w[from]) + from", &deps, &live, &shape, &[], None, None) + .unwrap(); + + assert_eq!( + partial, "PREVIOUS(sum(w[from])) + from", + "an index-nested live-source occurrence must not make the reducer hold \ + the live ref -- the whole reducer freezes and only the bare `from` \ + stays live; got: {partial}" + ); + // Belt-and-suspenders: the index-nested `from` must NOT be individually + // held live inside a recursed reducer. + assert!( + !partial.to_lowercase().contains("sum(previous(w"), + "the reducer must not recurse to hold the index-nested occurrence live; \ + got: {partial}" + ); +} + +/// Fig. 2 Q3 (Track A3 stage 2, review finding 2): an already-lagged other-dep +/// occurrence -- one that sits inside an ORIGINAL `PREVIOUS(...)` -- must be +/// left untouched (its live selection is suppressed AND it is never re-wrapped), +/// so the changed-first partial does not double-lag it to a t-2 read. +/// +/// `to = from + PREVIOUS(g)` with live source `from` (Bare): `from` stays live; +/// the already-lagged `PREVIOUS(g)` survives verbatim, NOT wrapped again as +/// `PREVIOUS(PREVIOUS(g))`. This complements +/// `partial_equation_does_not_rewrap_inside_previous` (which uses the +/// two-argument SAMPLE-IF-TRUE `PREVIOUS(target, input)` shape) with the exact +/// `to = from + PREVIOUS(g)` shape the stage-2 `occ.already_lagged` field must +/// reproduce. +#[test] +fn partial_leaves_already_lagged_other_dep_untouched() { + let deps = deps_set(&["from", "g"]); + let live = Ident::::new("from"); + let shape = RefShape::Bare; + + let partial = + build_partial_equation_shaped("from + PREVIOUS(g)", &deps, &live, &shape, &[], None, None) + .unwrap(); + + assert_eq!( + partial, "from + previous(g)", + "the already-lagged `PREVIOUS(g)` must survive verbatim and `from` must \ + stay live; got: {partial}" + ); + assert!( + !partial.to_lowercase().contains("previous(previous(g"), + "an already-lagged occurrence must not be double-wrapped; got: {partial}" + ); +} + // -- Arrayed-target link scores: per-element partial equations -- // // For a per-element-equation (`Ast::Arrayed`) target, the link score diff --git a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs index 2aa1391e9..c4cc22e7a 100644 --- a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs +++ b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs @@ -141,7 +141,10 @@ use super::*; use crate::ast::Ast; use crate::common::{Canonical, Ident}; -use crate::db::ltm_ir::{ReferenceSite, collect_all_reference_sites, model_ltm_reference_sites}; +use crate::db::ltm_ir::{ + OccurrenceRef, OccurrenceSite, ReferenceSite, collect_all_reference_sites, + model_ltm_reference_sites, +}; use crate::db::{ SimlinDb, project_dimensions_context, reconstruct_model_variables, sync_from_datamodel, }; @@ -441,6 +444,177 @@ fn pin_edge(compared: &ComparedShapes, from: &str, to: &str, expected: &[RefShap ); } +/// Assert the occurrence IR's per-occurrence stream for `to` aligns +/// position-for-position with the reparsed-Expr0 stream the ceteris-paribus +/// wrap runs on -- the corpus proof of the SiteId-zip bridge stage 2 consumes. +/// +/// Both streams are a left-to-right DFS over the target's slots (a scalar/A2A +/// target is one slot; an `Ast::Arrayed` target's slots are sorted then the +/// default). The IR stream (`model_ltm_reference_sites(..).occurrences[to]`) +/// comes from the Expr2 walk; the Expr0 stream from the production-printed, +/// reparsed text. Filtering the IR stream to `OccurrenceRef::Variable` (the +/// Expr0 walk records no module-qualified occurrence) leaves two streams that +/// must be equal length and, at each position, name the same source with the +/// same `in_reducer` context and -- off the reducer path -- the same shape. A +/// length or per-position mismatch is precisely the GH #913 / walker-divergence +/// desync a positional or SiteId zip must never silently tolerate, so it fails +/// loudly here. +fn assert_occurrence_stream_aligns( + to_str: &str, + to_var: &Variable, + variables: &HashMap, Variable>, + dim_ctx: &DimensionsContext, + ir: &crate::db::ltm_ir::LtmReferenceSitesResult, + expr0_occs: &[Expr0Occurrence], +) { + let ir_var_occs: Vec<&OccurrenceSite> = ir + .occurrences + .get(to_str) + .map(|v| { + v.iter() + .filter(|o| matches!(o.reference, OccurrenceRef::Variable(_))) + .collect() + }) + .unwrap_or_default(); + assert_eq!( + ir_var_occs.len(), + expr0_occs.len(), + "occurrence-stream LENGTH mismatch for target '{to_str}' (IR Expr2 walk vs reparsed \ + Expr0 walk): the per-occurrence SiteId zip the stage-2 deletion relies on would \ + desync.\n IR sources: {:?}\n Expr0 sources: {:?}", + ir_var_occs + .iter() + .map(|o| match &o.reference { + OccurrenceRef::Variable(s) => s.as_str(), + _ => "?", + }) + .collect::>(), + expr0_occs + .iter() + .map(|o| o.source.as_str()) + .collect::>(), + ); + for (ir_occ, e0_occ) in ir_var_occs.iter().zip(expr0_occs) { + let OccurrenceRef::Variable(ir_src) = &ir_occ.reference else { + unreachable!("filtered to Variable above"); + }; + assert_eq!( + ir_src, &e0_occ.source, + "occurrence-stream SOURCE mismatch for target '{to_str}' (IR vs reparsed Expr0)" + ); + assert_eq!( + ir_occ.in_reducer, e0_occ.in_reducer, + "occurrence-stream in_reducer mismatch for {ir_src} -> {to_str}" + ); + if !ir_occ.in_reducer { + let e0_shape = classify_expr0_occurrence(e0_occ, to_var, variables, dim_ctx); + assert_eq!( + ir_occ.shape, e0_shape, + "occurrence-stream SHAPE mismatch for {ir_src} -> {to_str}: the wrap's IR lookup \ + would return a shape the Expr0 classifier disagrees with" + ); + } + } +} + +/// Assert the occurrence-IR / reparsed-Expr0 stream alignment (the SiteId-zip +/// bridge) for EVERY target of `tp`'s `main` model, WITHOUT the per-edge +/// multiset gate. Lets the bridge be stress-tested on shapes the multiset gate +/// does not construct (module outputs, `PREVIOUS`/`INIT`-lagged references, +/// index-nested references, `LOOKUP` tables), where only the per-occurrence +/// alignment is the relevant property. +fn assert_occurrence_streams_align(tp: &TestProject) { + let datamodel = tp.build_datamodel(); + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, &datamodel); + let model = sync.models["main"].source; + let project = sync.project; + + let variables = reconstruct_model_variables(&db, model, project); + let dim_ctx = project_dimensions_context(&db, project); + let ir = model_ltm_reference_sites(&db, model, project); + + let mut to_names: Vec<&Ident> = variables.keys().collect(); + to_names.sort(); + for to_name in to_names { + let to_var = &variables[to_name]; + let expr0_occs = expr0_occurrences_for_target(to_name.as_str(), to_var, &variables); + assert_occurrence_stream_aligns( + to_name.as_str(), + to_var, + &variables, + dim_ctx, + ir, + &expr0_occs, + ); + } +} + +#[test] +fn align_already_lagged_and_index_nested_streams() { + // Occurrences inside `PREVIOUS(...)`/`INIT(...)` (already-lagged) and inside + // another reference's subscript index (index-nested) are the transform-only + // contexts the reports (fig2 Q3/Q4) flag as the subtle SiteId-zip cases. + // Both walkers must enumerate them in the SAME order for the zip to be sound. + let tp = TestProject::new("main") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .indexed_dimension("Slot", 4) + .array_aux("pop[Region]", "100") + .scalar_aux("g", "5") + .scalar_aux("idx", "1") + .array_aux("other[Slot]", "7") + .array_aux("lagged[Region]", "pop + PREVIOUS(g) + INIT(g)") + .array_aux("nested[Region]", "pop + other[idx]"); + assert_occurrence_streams_align(&tp); +} + +#[test] +fn align_reducer_and_lookup_streams() { + // A `LOOKUP` table argument is skipped by both walkers; reducer arguments + // are `in_reducer` on both; a multi-arg builtin (`MAX`) exercises the + // `walk_builtin_expr` (Expr2) vs positional (Expr0) child ordering the A2b + // bridge flags as the alignment subtlety. + let curve = datamodel::GraphicalFunction { + kind: datamodel::GraphicalFunctionKind::Continuous, + x_points: Some(vec![0.0, 10.0]), + y_points: vec![0.0, 5.0], + x_scale: datamodel::GraphicalFunctionScale { + min: 0.0, + max: 10.0, + }, + y_scale: datamodel::GraphicalFunctionScale { min: 0.0, max: 5.0 }, + }; + let tp = TestProject::new("main") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "100") + .scalar_aux("total", "SUM(pop[*])") + .scalar_aux("driver", "3") + .scalar_aux("g", "4") + .aux_with_gf("curve", "driver", curve) + .scalar_aux( + "looked_up", + "LOOKUP(curve, driver) + total + MAX(driver, g)", + ); + assert_occurrence_streams_align(&tp); +} + +#[test] +fn align_smooth_module_output_streams() { + // A SMOOTH expands to a module whose output is a `module·port` composite: + // the IR enumerates it as `OccurrenceRef::ModuleOutput` (filtered out of the + // Variable stream), while the Expr0 walk records no site for it. The + // remaining model-variable occurrences must still align, so the module + // occurrence does not desync the zip. + let tp = TestProject::new("main") + .with_sim_time(0.0, 5.0, 1.0) + .scalar_aux("level", "10") + .scalar_aux("other", "2") + .aux("smoothed", "SMTH1(level, 3) + other", None); + assert_occurrence_streams_align(&tp); +} + /// Drive both classifier families over every causal edge of `tp`'s `main` /// model and assert per-edge non-reducer shape agreement. /// @@ -503,6 +677,24 @@ fn assert_classifier_families_agree(tp: &TestProject) -> ComparedShapes { .push((shape, occ.in_reducer)); } + // Stage-2 groundwork: the per-occurrence SiteId-zip bridge, corpus-validated. + // + // Stage 2 replaces the ceteris-paribus wrap's Expr0 re-classification + // with a lookup into the occurrence IR (`OccurrenceSite`) by the + // occurrence's structural position, so there is ONE classifier family. + // That zip is sound only if the IR's per-occurrence stream + // (`occurrences[to]`, from the Expr2 walk) and the reparsed-Expr0 stream + // the wrap runs on ALIGN position-for-position. This is exactly the A2b + // bridge assumption, which was previously validated only on paper. Assert + // it here across every fixture: same length (no print/reparse GH #913 + // desync, no walker divergence), same source and `in_reducer` at each + // position, and -- for DIRECT (non-reducer) references -- the same shape + // the wrap would read from the IR instead of re-deriving. The reducer-arg + // `StarRange`->`Wildcard` AC1.4 asymmetry (the one deliberate divergence, + // module doc) is confined to `in_reducer` occurrences, so shape agreement + // is asserted only off the reducer path, matching the multiset gate above. + assert_occurrence_stream_aligns(to_str, to_var, &variables, dim_ctx, ir, &expr0_occs); + // Every source of an edge, from either side. let mut sources: std::collections::BTreeSet = Default::default(); sources.extend(expr2_sites.keys().cloned()); From 86df68bf01049370c03718c21117251dbb1ff2be Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 00:08:49 -0700 Subject: [PATCH 05/14] engine: drive LTM live-ref selection from the occurrence IR The ceteris-paribus wrap machine re-derived every per-occurrence decision -- live-shape matching, literal/element resolution, iterated-dim recognition, already-lagged skipping, element-selector and dimension-name index handling -- by classifying the printed-and-reparsed Expr0, mirroring the Expr2 classifiers under a "must stay in sync" docstring contract. Those decisions now come from the reference-site occurrence IR, threaded through the wrap with a structural path cursor whose alignment with the IR walker is proven corpus-wide by the gate's occurrence-alignment sweep. A missing or misaligned occurrence at any decision site degrades loudly (skip-and-warn), never to a silent guess. The sync contract is deleted. classify_expr0_subscript_shape, resolve_literal_element_index, is_live_source_iterated_dim_subscript, is_literal_element_index, and expr0_contains_live_match are gone from production (the first four survive cfg(test)-only, feeding synthetic non-compilable fixtures that cannot be built through a db); classify_other_dep_iterated_dim_subscript is deleted outright. The two lowering-side classifiers (classify_expr0_per_element_axes, expr0_iterated_axis_lines_up) survive lowering-only -- they classify the already-wrapped AST for row-pinning, which original-AST SiteIds cannot address -- behind a rustdoc'd seam that the typed-equation step deletes; wrap-time occurrence tagging was rejected because it would need a tag-only descent into PREVIOUS/INIT content the wrap never visits plus a Loc-keyed parallel map with its own drift surface. Every wrap call site threads a real occurrence stream; the empty- lookup shortcut and its incorrect justification are deleted. That also aligns the legacy and shaped link-score queries on the bare-live-source-inside-reducer class, where they had derived different equations (the compiled fragment came from one, the reported equation from the other): both now take the changed-last form, which is what shipped from both queries before this arc (the changed-first freeze form never shipped; pinned by legacy_and_shaped_bare_score_agree_when_source_bare_in_reducer). Un-hoisted-reducer and module-output-inside-reducer shapes keep their loud warned-skip degradation, pinned against the silent-zero variant. The characterization goldens are unchanged; the classifier-agreement gate becomes single-family classification pins plus the alignment sweep, with every expected-shape fixture retained. --- src/simlin-engine/src/db.rs | 39 +- src/simlin-engine/src/db/ltm/compile.rs | 12 + src/simlin-engine/src/db/ltm/link_scores.rs | 121 +++ .../scalar_feeder_bare_in_hoisted_reducer.txt | 4 + src/simlin-engine/src/db/ltm_char_tests.rs | 48 + src/simlin-engine/src/db/ltm_ir.rs | 61 +- src/simlin-engine/src/db/ltm_ir_tests.rs | 122 ++- src/simlin-engine/src/db/ltm_tests.rs | 46 + src/simlin-engine/src/db/ltm_unified_tests.rs | 110 ++ src/simlin-engine/src/db/tests.rs | 42 +- src/simlin-engine/src/ltm_augment.rs | 980 +++++++++++------- src/simlin-engine/src/ltm_augment_tests.rs | 145 ++- .../src/ltm_augment_wrap_test_support.rs | 273 +++++ .../src/ltm_classifier_agreement_tests.rs | 220 +++- 14 files changed, 1724 insertions(+), 499 deletions(-) create mode 100644 src/simlin-engine/src/db/ltm_char_golden/scalar_feeder_bare_in_hoisted_reducer.txt create mode 100644 src/simlin-engine/src/ltm_augment_wrap_test_support.rs diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index 9b5cf419f..b2070de6a 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -902,6 +902,22 @@ pub(crate) fn module_link_score_equation( let mut all_vars = HashMap::new(); all_vars.insert(to_ident.clone(), to_var.clone()); let dim_ctx = project_dimensions_context(db, project); + // The target's per-occurrence access-shape IR. The live source + // here is a `module·port` composite (an `OccurrenceRef::ModuleOutput`), + // so the ceteris-paribus wrap's GH #517 reducer-freeze arm + // (`subtree_has_live_shape`) must see it to reproduce the historical + // recursion when the composite is read bare inside a reducer + // (`to = SUM(arr[*] * module·port)`) -- an empty stream froze the + // reducer whole, silently converting a would-be loud degradation + // into a clean-compiling zero. This branch already read + // `model_ltm_reference_sites` (via `module_output_ref_in_document_order` + // above), so threading the stream adds no new salsa dependency. + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to_name) + .map(Vec::as_slice) + .unwrap_or(&[]); match crate::ltm_augment::generate_link_score_equation_for_link( &output_ident, &to_ident, @@ -913,6 +929,7 @@ pub(crate) fn module_link_score_equation( // Module-link partials are scalar-context; the GH #526 // other-dep check keeps its permissive legacy collapse. None, + to_occurrences, ) { Ok(eqn) => return Some(ltm::scalarize_ltm_equation(eqn)), // The target's equation couldn't be parsed for the @@ -987,6 +1004,23 @@ pub fn link_score_equation_text<'db>( all_vars.insert(from_ident.clone(), fv.clone()); } all_vars.insert(to_ident.clone(), to_var.clone()); + // The target's per-occurrence access-shape IR -- the SAME single classifier + // family the shaped twin (`link_score_equation_text_shaped`) threads. It is + // NOT optional: the ceteris-paribus wrap's GH #517 reducer-freeze arm + // (`subtree_has_live_shape`) consults the stream unconditionally, so passing + // an empty stream here (the pre-fix bug) made the wrap freeze a reducer whole + // where the shaped query recursed into it -- deriving a DIFFERENT partial for + // the very same scalar Bare score whenever the live source appears bare inside + // a reducer of the target's equation. Assembly compiles this legacy fragment + // while `model_ltm_variables` reports the shaped one, so the divergence made + // the VM simulate an equation that disagreed with the one reported. Threading + // the real stream single-sources the two. + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to_name) + .map(Vec::as_slice) + .unwrap_or(&[]); // A `PartialEquationError` here means the target's equation text could // not be parsed for the ceteris-paribus partial (GH #311). Skip the // link-score variable and surface a `Warning` instead of emitting a @@ -1002,8 +1036,11 @@ pub fn link_score_equation_text<'db>( None, // Legacy (from, to)-keyed path: no dims context is threaded at // all, so the GH #526 other-dep check keeps the permissive - // collapse here too. + // collapse here too. (This path is only consumed for a SCALAR target, + // whose empty iterated-dim space makes the other-dep verdict + // `NotIterated` regardless of `dep_dims`, so omitting it is sound.) None, + to_occurrences, ) { Ok(eqn) => eqn, Err(err) => { diff --git a/src/simlin-engine/src/db/ltm/compile.rs b/src/simlin-engine/src/db/ltm/compile.rs index 049adc8ba..fca781d99 100644 --- a/src/simlin-engine/src/db/ltm/compile.rs +++ b/src/simlin-engine/src/db/ltm/compile.rs @@ -300,6 +300,17 @@ pub fn link_score_equation_text_shaped<'db>( // would compile cleanly, so `model_ltm_fragment_diagnostics` would not // catch it, and degrading dependent loops to warned constant-0 stubs // would look like legitimate values. + // The target's per-occurrence access-shape IR (the single classifier + // family the ceteris-paribus wrap consumes). Empty for a target with no + // recorded occurrences (a structural edge); the wrap then makes no + // shape-driven decision. + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to_name) + .map(Vec::as_slice) + .unwrap_or(&[]); + let equation = match crate::ltm_augment::generate_link_score_equation_for_link( &from_ident, &to_ident, @@ -309,6 +320,7 @@ pub fn link_score_equation_text_shaped<'db>( &all_vars, Some(dim_ctx), Some(&dep_dims), + to_occurrences, ) { Ok(eqn) => eqn, Err(err) => { diff --git a/src/simlin-engine/src/db/ltm/link_scores.rs b/src/simlin-engine/src/db/ltm/link_scores.rs index 684d3ecfa..4dcf338d6 100644 --- a/src/simlin-engine/src/db/ltm/link_scores.rs +++ b/src/simlin-engine/src/db/ltm/link_scores.rs @@ -30,6 +30,57 @@ use super::parse::{ ltm_equation_dimensions, reconstruct_ltm_var_lowered, retarget_ltm_equation_dims, }; +/// The occurrence-stream slot each target element maps to when the +/// ceteris-paribus wrap walks one slot's expression at a time. +/// +/// `build_arrayed_link_score_equation` wraps each `Ast::Arrayed` slot +/// separately, so the per-element slots are numbered in canonical +/// element-key-sorted order (the SiteId slot prefix +/// [`crate::ltm_augment::OccurrenceLookup::for_slot`] strips) with the default +/// equation the slot AFTER the last element; a scalar / `ApplyToAll` target is a +/// single slot `0`. Shared by every per-element link-score emitter +/// (`try_scalar_to_arrayed_link_scores`, `emit_per_element_link_scores`, +/// `emit_agg_to_target_link_scores`) so the slot map and `OccurrenceLookup::for_slot` +/// cannot disagree. +struct ArrayedSlotMap { + /// `None` for a scalar / `ApplyToAll` target (one slot `0`); otherwise the + /// canonical-element-key -> slot map plus the default slot (`keys.len()`, + /// the slot after the last explicit element). + slots: Option<(HashMap, u16)>, +} + +impl ArrayedSlotMap { + fn new(ast: &crate::ast::Ast) -> Self { + use crate::ast::Ast; + let slots = if let Ast::Arrayed(_, per_elem, _, _) = ast { + let mut keys: Vec<&crate::common::CanonicalElementName> = per_elem.keys().collect(); + keys.sort(); + let map = keys + .iter() + .enumerate() + .map(|(i, k)| (k.as_str().to_string(), i as u16)) + .collect(); + Some((map, keys.len() as u16)) + } else { + None + }; + ArrayedSlotMap { slots } + } + + /// The slot for `element` (raw/unqualified -- the `per_elem` key form): its + /// own slot for an explicit `Ast::Arrayed` element, the default slot for an + /// unlisted one, or `0` for a scalar / `ApplyToAll` target. + fn slot_for(&self, element: &str) -> u16 { + match &self.slots { + None => 0, + Some((map, default_slot)) => { + let ce = crate::common::CanonicalElementName::from_raw(element); + map.get(ce.as_str()).copied().unwrap_or(*default_slot) + } + } + } +} + /// Determine the dimensions a link score should carry. /// /// Returns the target's dimension names when the edge is @@ -1178,6 +1229,20 @@ pub(super) fn try_scalar_to_arrayed_link_scores( // GH #910: resolved once for the whole target -- see `WithLookupSlotRefs`. let slot_refs = crate::ltm_augment::WithLookupSlotRefs::new(&to_var, target_ast_dims); + // The target's per-occurrence access-shape IR the ceteris-paribus wrap + // consumes (the single classifier family). `from` is a scalar model + // `Variable` that can appear bare inside a reducer of `to`'s equation, so + // the wrap's GH #517 reducer-freeze arm must see the real stream to recurse + // (matching the shaped per-shape path) instead of freezing the reducer whole + // -- an empty stream silently zeroed such a score. + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to) + .map(Vec::as_slice) + .unwrap_or(&[]); + let slot_map = ArrayedSlotMap::new(ast); + // Build one `LtmSyntheticVar` for `element` from its equation text and // that text's dependency set. The element name is the only part of the // generated equation/name that varies between elements. @@ -1218,6 +1283,10 @@ pub(super) fn try_scalar_to_arrayed_link_scores( // as `apply_implicit_with_lookup` leaves its value unwrapped. let gf_table_ref = slot_refs.for_element(&crate::common::CanonicalElementName::from_raw(element)); + let occ = crate::ltm_augment::OccurrenceLookup::for_slot( + to_occurrences, + slot_map.slot_for(element), + ); match crate::ltm_augment::generate_scalar_to_element_equation( from, to, @@ -1238,6 +1307,7 @@ pub(super) fn try_scalar_to_arrayed_link_scores( &[], Some(project_dimensions_context(db, project)), gf_table_ref.as_deref(), + &occ, ) { Ok(eqn) => eqn, Err(err) => { @@ -2324,6 +2394,16 @@ fn emit_per_element_link_scores( let slot_refs = crate::ltm_augment::WithLookupSlotRefs::new(&to_var, target_ast_dims); let dim_ctx = project_dimensions_context(db, project); + // The target's per-occurrence access-shape IR the ceteris-paribus wrap + // consumes (the single classifier family). + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to) + .map(Vec::as_slice) + .unwrap_or(&[]); + let slot_map = ArrayedSlotMap::new(ast); + // Arrayed deps sharing a target dim get element-pinned in the scalar // per-element equation (mirroring `try_scalar_to_arrayed_link_scores`); // the source itself is excluded -- its occurrences are pinned per-row @@ -2458,6 +2538,10 @@ fn emit_per_element_link_scores( if !emitted.insert(name.clone()) { continue; } + let occ = crate::ltm_augment::OccurrenceLookup::for_slot( + to_occurrences, + slot_map.slot_for(element), + ); match crate::ltm_augment::generate_per_element_link_equation( from, to, @@ -2472,6 +2556,7 @@ fn emit_per_element_link_scores( &target_iterated_dims, dim_ctx, gf_table_ref.as_deref(), + &occ, ) { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, @@ -3086,6 +3171,24 @@ pub(super) fn emit_agg_to_target_link_scores( Ast::Scalar(_) => &[], Ast::ApplyToAll(dims, _) | Ast::Arrayed(dims, _, _, _) => dims, }; + // The target slot's occurrence stream the ceteris-paribus wrap consumes -- + // the SAME single classifier family every other emitter threads. The live + // source here is the synthetic agg (`$⁚ltm⁚agg⁚n`), which is never a recorded + // occurrence (it is held live by `reducer_subst` text-matching, and during + // the wrap the reducer that became it is still spelled `SUM(...)`), so the + // wrap's `subtree_has_live_shape`/`get` for the agg never hit and the stream + // is behavior-neutral for it -- exactly why the retired `OccurrenceLookup::empty()` + // was byte-equivalent here. Threading the real stream instead removes the + // empty-stream constructor (and the "my source is never in a reducer" footgun + // that mis-fired at the three scalar/module sites) rather than paying it with + // a parallel code path. + let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); + let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites + .occurrences + .get(to) + .map(Vec::as_slice) + .unwrap_or(&[]); + let slot_map = ArrayedSlotMap::new(ast); // GH #910: when `to` is an implicit WITH-LOOKUP variable, the per-element // partials below are full re-evaluations of its RAW element equation (with // the reducer substituted by the agg name), while the guard form ratios them @@ -3352,6 +3455,10 @@ pub(super) fn emit_agg_to_target_link_scores( // GH #910: a scalar with-lookup target's single table, // referenced by bare ident. crate::ltm_augment::with_lookup_table_ref(&to_var).as_deref(), + // Agg source: the live thing is the synthetic agg, never a + // recorded occurrence, so the real stream is behavior-neutral + // (the wrap's agg lookups all miss). A scalar target is slot 0. + &crate::ltm_augment::OccurrenceLookup::for_slot(to_occurrences, 0), ) { Ok(equation) => vars.push(LtmSyntheticVar { name, @@ -3413,6 +3520,13 @@ pub(super) fn emit_agg_to_target_link_scores( &source_pins_for_target(element), Some(project_dimensions_context(db, project)), elem_gf_ref(element).as_deref(), + // Agg source: the live thing is the synthetic agg, never a + // recorded occurrence, so the real stream is behavior-neutral + // (the wrap's agg lookups all miss). A2A body is slot 0. + &crate::ltm_augment::OccurrenceLookup::for_slot( + to_occurrences, + slot_map.slot_for(element), + ), ) { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, @@ -3488,6 +3602,13 @@ pub(super) fn emit_agg_to_target_link_scores( &source_pins_for_target(element), Some(project_dimensions_context(db, project)), elem_gf_ref(element).as_deref(), + // Agg source: the live thing is the synthetic agg, + // never a recorded occurrence, so the real stream is + // behavior-neutral (the wrap's agg lookups all miss). + &crate::ltm_augment::OccurrenceLookup::for_slot( + to_occurrences, + slot_map.slot_for(element), + ), ) } }; diff --git a/src/simlin-engine/src/db/ltm_char_golden/scalar_feeder_bare_in_hoisted_reducer.txt b/src/simlin-engine/src/db/ltm_char_golden/scalar_feeder_bare_in_hoisted_reducer.txt new file mode 100644 index 000000000..6a17a9c33 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/scalar_feeder_bare_in_hoisted_reducer.txt @@ -0,0 +1,4 @@ +$⁚ltm⁚link_score⁚scale→$⁚ltm⁚agg⁚0 +scalar: if (TIME = INITIAL_TIME) then 0 else if (("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0")) = 0) OR ((scale - PREVIOUS(scale)) = 0) then 0 else SAFEDIV(("$⁚ltm⁚agg⁚0" - (sum(arr[*] * PREVIOUS(scale)))), ABS(("$⁚ltm⁚agg⁚0" - PREVIOUS("$⁚ltm⁚agg⁚0"))), 0) * SIGN((scale - PREVIOUS(scale))) +$⁚ltm⁚link_score⁚scale→total +scalar: if (TIME = INITIAL_TIME) then 0 else if ((total - PREVIOUS(total)) = 0) OR ((scale - PREVIOUS(scale)) = 0) then 0 else SAFEDIV((total - (PREVIOUS(scale) + sum(arr[*] * PREVIOUS(scale)))), ABS((total - PREVIOUS(total))), 0) * SIGN((scale - PREVIOUS(scale))) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index aec08ace0..e7d8387d4 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -1276,3 +1276,51 @@ fn char_already_lagged_other_dep() { ); assert_golden("already_lagged_other_dep", &actual); } + +// --------------------------------------------------------------------------- +// Model H (Track A3 stage 2b, finding 1): a SCALAR feeder read BARE both +// OUTSIDE and INSIDE a HOISTED reducer of a SCALAR target. +// +// `total = scale + SUM(arr[*] * scale)`: `scale` appears bare (outside the +// reducer) AND as the reducer's scalar feeder inside `SUM(arr[*] * scale)`. The +// `scale -> total` Bare link score is the finding-1 probe. With an empty +// occurrence stream the LEGACY `link_score_equation_text` (which assembly +// compiles) froze the whole `SUM(...)` reducer -- changed-FIRST -- while the +// SHAPED emitter (which `model_ltm_variables` reports/serializes) threaded the +// real stream and recursed into it -- changed-LAST. So the COMPILED fragment +// disagreed with the REPORTED equation: the VM simulated one derivation and the +// report showed another. Threading the real stream makes the legacy query +// changed-LAST too. This golden pins the emitted (changed-LAST) text -- the +// numerator subtracts `PREVIOUS(scale) + sum(arr[*] * PREVIOUS(scale))` from the +// live `total`, i.e. `scale` is held live in BOTH occurrences -- and +// `legacy_and_shaped_bare_score_agree_when_source_bare_in_reducer` (in +// `db::ltm_tests`) pins that the legacy (compiled) query now returns the +// identical bytes. GH #517/#743: the changed-LAST convention is the correct one +// (freeze the target's OTHER inputs, hold the scored source live everywhere it +// appears), so the drift is resolved onto the shaped semantics, not HEAD's. +// --------------------------------------------------------------------------- + +fn scalar_feeder_bare_in_hoisted_reducer_model() -> datamodel::Project { + TestProject::new("scalar_bare_in_reducer_char") + .with_sim_time(0.0, 10.0, 1.0) + .named_dimension("D1", &["a", "b"]) + .array_aux("arr[D1]", "1") + .aux("scale", "pop * 0.01", None) + .aux("total", "scale + SUM(arr[*] * scale)", None) + .flow("growth", "total * 0.001", None) + .stock("pop", "1", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_scalar_feeder_bare_in_hoisted_reducer() { + // Every `scale -> X` score: the site-1 `scale -> total` Bare score (both + // occurrences held live, changed-LAST) plus the `scale -> $⁚ltm⁚agg⁚0` + // scalar-feeder score, so the whole feeder attribution is frozen. + let actual = dump_synthetic_vars( + scalar_feeder_bare_in_hoisted_reducer_model(), + true, + "link_score\u{205A}scale", + ); + assert_golden("scalar_feeder_bare_in_hoisted_reducer", &actual); +} diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index d22172c2d..dfa6a202b 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -663,6 +663,29 @@ fn walk_all_in_expr( index_nested, ); } else if let Some((module, port)) = module_output_parts(ident.as_str(), ctx) { + // Classify a subscripted module output's axes the SAME way a + // model-variable subscript's are (empty `from_dims`, since a + // `module·port` composite is not a variable key -- so a bare + // iterated-dim index lands `MismatchedIterated`, an over-arity + // or non-iterated index `Dynamic`). This preserves byte parity + // with the retired Expr0 `classify_other_dep_iterated_dim_subscript`, + // which likewise built iterated axes for ANY subscripted + // iterated-dim head: the wrap's `other_dep_verdict` on those + // axes (dep arity always `None` for a non-variable head) then + // permissively collapses `mod·out[Region]` to `PREVIOUS(mod·out)` + // rather than freezing the uncompilable dim-name subscript. An + // empty `axes` here (the pre-flip stage-2 state) instead derived + // `NotIterated`, a silent divergence on an arrayed user-module + // output referenced by an iterated subscript. Pinned at the IR + // level by `subscripted_arrayed_module_output_axes_derive_collapse`; + // no simulate-corpus fixture exercises the end-to-end score, so + // both LTM suites stayed byte-green either way. + let axes = classify_occurrence_axes( + indices, + &from_dims, + &ctx.target_iterated_dims, + ctx.dim_ctx, + ); acc.push_occurrence( OccurrenceRef::ModuleOutput { module, @@ -670,7 +693,7 @@ fn walk_all_in_expr( composite: ident.as_str().to_string(), }, RefShape::Bare, - Vec::new(), + axes, target_element, reducer_keys, already_lagged, @@ -1089,37 +1112,19 @@ pub(crate) enum OtherDepVerdict { /// absent from the variable map / has no declared dims) and the target's /// iterated-dim count `target_iterated_count`. /// -/// `ltm_augment::classify_other_dep_iterated_dim_subscript` (the Expr0-side -/// classifier the ceteris-paribus wrap consults) builds `axes` from the parsed -/// subscript and delegates here. That wrap-side axis construction is still a -/// textual mirror of the IR walker's [`classify_occurrence_axes`] -- Track A3 -/// stage 2 has not yet retargeted the wrap onto the occurrence stream -- but the -/// two derivations provably cannot produce a different VERDICT, the only value -/// the wrap consumes: -/// -/// - Every IN-arity position (index `< dep_arity`) agrees: both derivations gate -/// the iterated-dim lineup on the identical [`crate::ltm_agg::iterated_axis_slot_elements`] -/// (compare [`crate::ltm_agg::classify_axis_access`]'s `Var` arm with -/// `ltm_augment::other_dep_axis_lines_up`), so they label each such index the -/// same `Iterated` / `MismatchedIterated`. -/// - The two disagree ONLY on an OVER-declared-arity index (index `>= dep_arity`): -/// the mirror's `dep_dims.get(i) == None` arm labels it `Iterated{d,d}`, the -/// IR's `source_dims.get(i) == None` arm labels it `MismatchedIterated{d}`. -/// But such a position exists only when `axes.len() > dep_arity`, and the -/// arity check below returns `Mismatch` for that whole case BEFORE inspecting -/// the per-axis arms. +/// The ceteris-paribus wrap (`ltm_augment::other_dep_verdict`) reads a +/// non-live-dep subscript occurrence's `axes` straight off the occurrence IR +/// (via `OccurrenceLookup`) and calls this -- there is no longer an Expr0-side +/// re-derivation of the verdict on the live path, so the wrap and the edge +/// emitter cannot drift: they consume the SAME classification. (The Expr0 +/// `axes` builder survives only `#[cfg(test)]`, to reconstruct occurrences for +/// the text-level wrap unit tests, and is proven in step with `classify_occurrence_axes` +/// by the alignment gate.) /// -/// So the arity guard DOMINATES the sole labeling difference: the silent-zero -/// drift the two-family "must stay in sync" contract guarded against is -/// structurally impossible for this verdict, whichever family built `axes`. -/// Stage 2 will still delete the wrap-side axis construction (collapsing to one -/// classifier family), but the verdict itself is already single-sourced here. /// See [`OccurrenceAxis`]'s rustdoc for the full rule; the two load-bearing /// arity corners (under-arity all-`Iterated` => `Mismatch`; over-target-arity => /// `NotIterated`) are pinned by `under_arity_iterated_subscript_is_mismatch_not_collapse` -/// / `over_target_arity_iterated_subscript_is_not_iterated`, and the -/// dominance argument by `verdict_ignores_over_arity_axis_labeling`, in -/// `db::ltm_ir_tests`. +/// / `over_target_arity_iterated_subscript_is_not_iterated` in `db::ltm_ir_tests`. pub(crate) fn derive_other_dep_verdict( axes: &[OccurrenceAxis], dep_arity: Option, diff --git a/src/simlin-engine/src/db/ltm_ir_tests.rs b/src/simlin-engine/src/db/ltm_ir_tests.rs index 277b2a710..a5c0e4f95 100644 --- a/src/simlin-engine/src/db/ltm_ir_tests.rs +++ b/src/simlin-engine/src/db/ltm_ir_tests.rs @@ -1098,8 +1098,8 @@ mod occurrence_ir_tests { // A transposed subscript `arr[D2, D1]` (arr declared `[D1, D2]`) inside // an A2A-over-`[D1, D2]` target: the coarse shape collapses to // DynamicIndex (unchanged), but the per-axis record marks each axis - // `MismatchedIterated`, so `classify_other_dep_iterated_dim_subscript`'s - // `Mismatch` verdict is derivable from the IR (distinct from a genuine + // `MismatchedIterated`, so `derive_other_dep_verdict`'s + // `Mismatch` is derivable from the IR (distinct from a genuine // dynamic index below). let project = TestProject::new("main") .named_dimension("D1", &["a", "b"]) @@ -1241,6 +1241,96 @@ mod occurrence_ir_tests { ); } + /// An ARRAYED user-module output referenced by an iterated-dim subscript + /// as a NON-live dep of an A2A target (finding 3 / byte-parity restore). + /// The walker classifies a subscripted `module·port` composite's axes + /// EXACTLY like a model-variable subscript's: `module·port` is never a + /// variable key, so `from_dims` is empty and a bare iterated-dim index + /// (`Region`) lands `MismatchedIterated`. With a non-variable head the dep + /// arity is `None`, so `derive_other_dep_verdict` permissively COLLAPSES + /// the subscript -- the ceteris-paribus wrap then rewrites + /// `mod·out[Region]` to a bare `PREVIOUS(mod·out)`, matching the retired + /// Expr0 `classify_other_dep_iterated_dim_subscript`. The pre-fix stage-2 + /// state pushed EMPTY `axes` for a module-output subscript, which derived + /// `NotIterated` and froze the uncompilable dim-name subscript verbatim -- + /// a silent divergence no corpus fixture covered (both suites stayed + /// byte-green either way). + #[test] + fn subscripted_arrayed_module_output_axes_derive_collapse() { + use datamodel::{Aux, Equation, Variable}; + let arrayed_aux = |ident: &str, eqn: &str, can_input: bool| -> Variable { + Variable::Aux(Aux { + ident: ident.to_string(), + equation: Equation::ApplyToAll(vec!["Region".to_string()], eqn.to_string()), + documentation: String::new(), + units: None, + gf: None, + ai_state: None, + uid: None, + compat: datamodel::Compat { + can_be_module_input: can_input, + ..datamodel::Compat::default() + }, + }) + }; + let project = datamodel::Project { + name: "arrayed_mod".to_string(), + sim_specs: datamodel::SimSpecs { + start: 0.0, + stop: 1.0, + dt: datamodel::Dt::Dt(1.0), + save_step: None, + sim_method: datamodel::SimMethod::Euler, + time_units: None, + }, + dimensions: vec![datamodel::Dimension::named( + "Region".to_string(), + vec!["nyc".to_string(), "boston".to_string()], + )], + units: vec![], + models: vec![ + x_model( + "main", + vec![ + arrayed_aux("live", "1", false), + // A2A over `Region`, referencing the arrayed module + // output by an iterated-dim subscript as a non-live dep. + arrayed_aux("combined", "live[Region] + sub.out[Region]", false), + x_module("sub", &[("live", "sub.input")], None), + ], + ), + x_model( + "sub", + vec![ + arrayed_aux("input", "0", true), + arrayed_aux("out", "input[Region] * 2", false), + ], + ), + ], + source: None, + ai_information: None, + }; + let occs = occ_from_datamodel(&project, "main", "combined"); + let mod_occ = occs + .iter() + .find(|o| matches!(&o.reference, OccurrenceRef::ModuleOutput { .. })) + .unwrap_or_else(|| { + panic!("expected a ModuleOutput occurrence for sub·out; occs: {occs:?}") + }); + assert!( + !mod_occ.axes.is_empty(), + "a subscripted module output must carry classified axes (not empty), \ + else the verdict silently derives NotIterated: {mod_occ:?}" + ); + // Dep arity is `None` (a `module·port` composite is not a variable key), + // so the verdict permissively collapses -- byte-parity with HEAD. + assert_eq!( + derive_other_dep_verdict(&mod_occ.axes, None, 1), + OtherDepVerdict::Collapse, + "an iterated-dim subscript on an unthreadable module output collapses" + ); + } + /// The dominant production shape: an IMPLICIT stdlib expansion. `smoothed = /// SMTH1(input, 5) * 2` desugars (in the builtins visitor) to an implicit /// `Variable::Module` named `$⁚smoothed⁚0⁚smth1` whose `·output` composite @@ -1378,8 +1468,8 @@ mod occurrence_ir_tests { // ── Other-dep verdict derivation (the contract on `axes`) ────────────── // // `OccurrenceAxis`'s rustdoc states the rule by which the transform derives - // `ltm_augment::classify_other_dep_iterated_dim_subscript`'s - // `Collapse`/`Mismatch`/`NotIterated` verdict from an occurrence's `axes`. + // `derive_other_dep_verdict`'s + // `Collapse`/`Mismatch`/`NotIterated` from an occurrence's `axes`. // Track A3 stage 2 promoted the reference derivation to the production // `db::ltm_ir::derive_other_dep_verdict` -- the Expr0-side classifier now // builds `axes` from the parsed subscript and DELEGATES to it, so the wrap @@ -1461,8 +1551,8 @@ mod occurrence_ir_tests { // A2A-over-[D1,D2] target. The single index lines up with arr's first // axis, so the occurrence's axes are all-`Iterated` -- yet the dep's // declared arity is 2, so - // `classify_other_dep_iterated_dim_subscript` returns `Mismatch` - // (ltm_augment.rs:288, `index_dims.len() != dep_dims.len()`), NOT the + // `derive_other_dep_verdict` returns `Mismatch` + // (`axes.len() != dep_arity`, checked before the per-axis arms), NOT the // Collapse a rule keyed on the per-axis arms alone would derive. // Freezing the wrong element here is the GH #526 silent magnitude error. let project = TestProject::new("main") @@ -1503,8 +1593,8 @@ mod occurrence_ir_tests { // A2A-over-[D1] target. Position 0 lines up (`Iterated`); position 1's // `D1` names the source's `D2` axis, so it is `MismatchedIterated`. // But the subscript has MORE indices (2) than the target has iterated - // dims (1), so `classify_other_dep_iterated_dim_subscript` short-circuits - // to `NotIterated` (ltm_augment.rs:268, normal wrap), NOT the Mismatch + // dims (1), so `derive_other_dep_verdict` short-circuits + // to `NotIterated` (`axes.len() > target_iterated_count`), NOT the Mismatch // the `MismatchedIterated` arm alone would derive. let project = TestProject::new("main") .named_dimension("D1", &["a", "b"]) @@ -1536,8 +1626,8 @@ mod occurrence_ir_tests { #[test] fn verdict_ignores_over_arity_axis_labeling() { // Track A3 stage 2 / finding-3 verdict-equivalence. The Expr0-side - // mirror (`ltm_augment::classify_other_dep_iterated_dim_subscript`) and - // the IR walker (`classify_occurrence_axes`) can LABEL a per-axis index + // `#[cfg(test)]` axis builder (`ltm_augment::other_dep_occurrence_axes`) + // and the IR walker (`classify_occurrence_axes`) can LABEL a per-axis index // differently at exactly ONE kind of position: an index that overflows // the dep's declared arity. The mirror's `dep_dims.get(i) == None` arm // marks it `Iterated{d,d}`; the IR's `source_dims.get(i) == None` arm @@ -1550,11 +1640,13 @@ mod occurrence_ir_tests { // dep_arity`, and the arity check in `derive_other_dep_verdict` returns // `Mismatch` for that whole case BEFORE inspecting the per-axis arms. So // the sole labeling difference is dominated: the verdict cannot differ - // between the two derivations. That is what makes the shared verdict - // rule single-sourced -- the "silent drift" the two-family "must stay in - // sync" contract guards against is structurally impossible for the - // VERDICT (the only value the wrap consumes), whichever family built the - // `axes`. + // between the two derivations. Production now reads `axes` straight off + // the occurrence IR (one classifier family), so this only guards the + // `#[cfg(test)]` Expr0 axis builder (`other_dep_occurrence_axes`) that + // reconstructs occurrences for the text-level wrap unit tests: it stays + // VERDICT-equivalent to `classify_occurrence_axes` even where the two + // label an over-arity axis differently, so the reconstructed occurrence + // is a faithful stand-in. // // 3 indices, dep declared arity 2 (index 2 overflows), target // iterated-dim count 3. diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index b0972a59c..fb5bbec4c 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -748,3 +748,49 @@ fn collect_agg_petals_groups_single_agg_circuits() { assert_eq!(p.nodes[0], agg, "petal rotated to start at the agg"); } } + +/// Finding-1 regression: the `(from, to)`-keyed legacy `link_score_equation_text` +/// and the shape-aware `link_score_equation_text_shaped(.., Bare)` must derive +/// the SAME equation for a standard scalar Bare link score. The two twins had +/// silently drifted: the legacy query passed an EMPTY occurrence stream, so the +/// ceteris-paribus wrap's GH #517 reducer-freeze arm (`subtree_has_live_shape`) +/// froze a reducer whole, while the shaped query threaded the real stream and +/// recursed into it -- producing a different partial for `scale -> total` +/// whenever the live source `scale` appears bare inside a reducer of `total`. +/// The compiled fragment (assembly routes the standard scalar Bare score through +/// the legacy query) would then simulate an equation that disagrees with the one +/// `model_ltm_variables` reports/serializes. +#[test] +fn legacy_and_shaped_bare_score_agree_when_source_bare_in_reducer() { + let project = TestProject::new("bare_source_in_reducer") + .with_sim_time(0.0, 10.0, 1.0) + .named_dimension("D1", &["a", "b"]) + .array_aux("arr[D1]", "1") + .aux("scale", "pop * 0.01", None) + .aux("total", "scale + SUM(arr[*] * scale)", None) + .flow("growth", "total * 0.001", None) + .stock("pop", "1", &["growth"], &[], None) + .build_datamodel(); + + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, &project); + let source_model = sync.models["main"].source; + + let link_id = LtmLinkId::new(&db, "scale".to_string(), "total".to_string()); + let legacy = link_score_equation_text(&db, link_id, source_model, sync.project) + .as_ref() + .expect("legacy scale -> total link score should exist"); + let shaped = + link_score_equation_text_shaped(&db, link_id, RefShape::Bare, source_model, sync.project); + let ShapedLinkScore::Scored(shaped) = shaped else { + panic!("shaped scale -> total Bare link score should be Scored"); + }; + + assert_eq!( + legacy.equation.source_text(), + shaped.equation.source_text(), + "the legacy (compiled/assembled) and shaped (emitted/reported) Bare link \ + scores must be byte-identical -- a divergence means the VM simulates a \ + different equation than is reported" + ); +} diff --git a/src/simlin-engine/src/db/ltm_unified_tests.rs b/src/simlin-engine/src/db/ltm_unified_tests.rs index c6fbdf0dd..83f18c248 100644 --- a/src/simlin-engine/src/db/ltm_unified_tests.rs +++ b/src/simlin-engine/src/db/ltm_unified_tests.rs @@ -70,6 +70,116 @@ fn test_model_ltm_variables_stdlib_module() { ); } +/// Finding-2 regression (scalar source -> arrayed target, un-hoisted reducer): +/// `try_scalar_to_arrayed_link_scores` -> `generate_scalar_to_element_equation` +/// used an EMPTY occurrence stream, so the ceteris-paribus wrap's GH #517 +/// reducer-freeze arm froze the reducer whole even when the scalar live source +/// `scale` appears bare inside it. That silently converted a would-be loud +/// degradation (HEAD recursed into the reducer, producing an uncompilable +/// `PREVIOUS`-of-wildcard-slice fragment that surfaces as a `model_ltm_fragment_diagnostics` +/// Warning) into a clean-compiling silent zero holding no live occurrence. +/// Threading the real per-slot occurrence stream restores the recursion, so the +/// edge degrades LOUDLY (the repo's standard) instead. +#[test] +fn scalar_to_arrayed_reducer_source_recurses_and_degrades_loudly() { + use salsa::Setter; + let mut db = SimlinDb::default(); + let project = TestProject::new("scalar_to_arrayed_reducer") + .with_sim_time(0.0, 10.0, 1.0) + .named_dimension("D1", &["a", "b"]) + .named_dimension("D2", &["c", "d"]) + .aux("idx", "1", None) + .aux("scale", "pop * 0.01", None) + .array_aux("pop2[D1,D2]", "pop") + // Dynamic index `idx` keeps the reducer un-hoisted, routing the + // `scale -> share` edge through `try_scalar_to_arrayed_link_scores`. + // `scale` appears ONLY inside the reducer. + .array_aux("share[D1]", "1 + SUM(pop2[idx,*] * scale)") + .flow("growth", "SUM(share[*]) * 0.001", None) + .stock("pop", "1", &["growth"], &[], None) + .build_datamodel(); + let sync = sync_from_datamodel(&db, &project); + sync.project.set_ltm_enabled(&mut db).to(true); + let model = sync.models["main"].source; + + let ltm = model_ltm_variables(&db, model, sync.project); + let share_score = ltm + .vars + .iter() + .find(|v| v.name.contains("scale") && v.name.contains("share")) + .expect("scale -> share[a] link score should be emitted"); + let eqn = share_score.equation.source_text(); + // The reducer is recursed into (its `scale`-frozen inner slice re-prints + // as `previous(pop2[...])`), NOT frozen whole as `previous(sum(...))`. + assert!( + eqn.contains("previous(pop2"), + "the reducer must be recursed into (other-dep frozen inside it), got: {eqn}" + ); + assert!( + !eqn.to_lowercase().contains("previous(sum("), + "the reducer must NOT be frozen whole, got: {eqn}" + ); + + // The recursed partial is an uncompilable `PREVIOUS`-of-wildcard-slice, so + // assembly surfaces a LOUD fragment-diagnostics Warning rather than a silent + // clean-compiling zero. + let diags = collect_model_diagnostics(&db, model, sync.project); + assert!( + diags.iter().any(|d| matches!( + &d.error, + crate::db::DiagnosticError::Assembly(m) + if m.contains("scale") && m.contains("share") && m.contains("failed to compile") + )), + "a loud fragment-compile Warning should name the scale -> share edge; got {:?}", + diags.iter().map(|d| &d.error).collect::>() + ); +} + +/// Finding-2 regression (module output read bare inside a reducer): +/// `module_link_score_equation`'s `module -> variable` branch passed an EMPTY +/// occurrence stream AND `subtree_has_live_shape` matched only `Variable` +/// occurrences, so a module-output composite read bare inside a reducer +/// (`total = SUM(arr[*] * SMTH1(x, 3))`) froze the reducer whole -- dropping the +/// live occurrence and silently zeroing the score. Threading the real stream and +/// teaching `subtree_has_live_shape` to also match `ModuleOutput` composites +/// restores HEAD's recursion: the changed-last partial keeps the reducer live +/// and freezes only the (scalar) module output, so the score is real. +#[test] +fn module_output_bare_in_reducer_recurses() { + use salsa::Setter; + let mut db = SimlinDb::default(); + let project = TestProject::new("mod_output_in_reducer") + .with_sim_time(0.0, 10.0, 1.0) + .named_dimension("D1", &["a", "b"]) + .array_aux("arr[D1]", "1") + .aux("x", "pop * 0.5", None) + .aux("total", "SUM(arr[*] * SMTH1(x, 3))", None) + .flow("growth", "total * 0.001", None) + .stock("pop", "1", &["growth"], &[], None) + .build_datamodel(); + let sync = sync_from_datamodel(&db, &project); + sync.project.set_ltm_enabled(&mut db).to(true); + let model = sync.models["main"].source; + + let ltm = model_ltm_variables(&db, model, sync.project); + let mod_score = ltm + .vars + .iter() + .find(|v| v.name.contains("smth1") && v.name.ends_with("total")) + .expect("smth1 -> total module link score should be emitted"); + let eqn = mod_score.equation.source_text(); + // The reducer is kept live (recursed) with only the scalar module output + // frozen (`sum(arr[*] * PREVIOUS(...·output))`), NOT frozen whole. + assert!( + eqn.contains("\u{B7}output") && eqn.to_lowercase().contains("sum(arr[*]"), + "the module-output reducer must be recursed into, got: {eqn}" + ); + assert!( + !eqn.to_lowercase().contains("previous(sum("), + "the reducer must NOT be frozen whole, got: {eqn}" + ); +} + #[test] fn test_model_ltm_variables_passthrough_module() { let db = SimlinDb::default(); diff --git a/src/simlin-engine/src/db/tests.rs b/src/simlin-engine/src/db/tests.rs index 0421b9cdd..2e48fe459 100644 --- a/src/simlin-engine/src/db/tests.rs +++ b/src/simlin-engine/src/db/tests.rs @@ -2089,6 +2089,17 @@ fn two_loop_project() -> datamodel::Project { } } +/// Per-link incrementality: editing loop A's variable must NOT force +/// recompilation of loop B's link-score bytecode FRAGMENT. +/// +/// The meaningful cache is the compiled fragment (`compile_ltm_var_fragment`), +/// not the equation-text query. Since the finding-1 fix, the equation-text +/// query `link_score_equation_text` reads the whole-model occurrence IR +/// (`model_ltm_reference_sites`) so the compiled fragment matches the emitted +/// one -- so editing ANY variable re-runs it. But it produces an UNCHANGED value +/// for an unaffected edge, so salsa backdates it and the expensive +/// `compile_ltm_var_fragment` dependent is NOT re-executed: the fragment stays +/// pointer-stable. That fragment-level reuse is what this test guards. #[test] fn test_ltm_per_link_caching() { use salsa::Setter; @@ -2108,17 +2119,14 @@ fn test_ltm_per_link_caching() { // so that the immutable borrow on db is released before the mutation. let (link_b_ptr_before, link_a_ptr_before) = { let link_b_id = LtmLinkId::new(&db, "stock_b".to_string(), "births_b".to_string()); - let link_b_before = link_score_equation_text(&db, link_b_id, source_model, source_project); - assert!(link_b_before.is_some(), "link B score should exist"); + let link_b_before = compile_ltm_var_fragment(&db, link_b_id, source_model, source_project); + assert!(link_b_before.is_some(), "link B fragment should exist"); let link_a_id = LtmLinkId::new(&db, "stock_a".to_string(), "births_a".to_string()); - let link_a_before = link_score_equation_text(&db, link_a_id, source_model, source_project); - assert!(link_a_before.is_some(), "link A score should exist"); + let link_a_before = compile_ltm_var_fragment(&db, link_a_id, source_model, source_project); + assert!(link_a_before.is_some(), "link A fragment should exist"); - ( - link_b_before as *const Option, - link_a_before as *const Option, - ) + (link_b_before as *const _, link_a_before as *const _) }; // Change births_a equation (affects loop A, should NOT affect loop B) @@ -2132,20 +2140,22 @@ fn test_ltm_per_link_caching() { let link_b_id = LtmLinkId::new(&db, "stock_b".to_string(), "births_b".to_string()); let link_a_id = LtmLinkId::new(&db, "stock_a".to_string(), "births_a".to_string()); - // Link B should be pointer-equal (cached) since births_b is unaffected - let link_b_after = link_score_equation_text(&db, link_b_id, source_model, source_project); - let link_b_ptr_after = link_b_after as *const Option; + // Link B's FRAGMENT should be pointer-equal (backdate-cached): its + // equation-text query re-ran but produced an unchanged value, so the + // fragment compile was not re-executed. + let link_b_after = compile_ltm_var_fragment(&db, link_b_id, source_model, source_project); + let link_b_ptr_after = link_b_after as *const _; assert_eq!( link_b_ptr_before, link_b_ptr_after, - "link score for unaffected loop B should be cached (pointer-equal)" + "fragment for unaffected loop B should be cached (pointer-equal)" ); - // Link A should be recomputed (equation changed for births_a) - let link_a_after = link_score_equation_text(&db, link_a_id, source_model, source_project); - let link_a_ptr_after = link_a_after as *const Option; + // Link A's fragment should be recomputed (equation changed for births_a) + let link_a_after = compile_ltm_var_fragment(&db, link_a_id, source_model, source_project); + let link_a_ptr_after = link_a_after as *const _; assert_ne!( link_a_ptr_before, link_a_ptr_after, - "link score for affected loop A should be recomputed" + "fragment for affected loop A should be recomputed" ); } diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 68475f9a4..a1e1b44f8 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -20,7 +20,130 @@ use crate::variable::{Variable, identifier_set}; use std::collections::{HashMap, HashSet}; use crate::db::RefShape; -use crate::db::ltm_ir::{OccurrenceAxis, OtherDepVerdict, derive_other_dep_verdict}; +use crate::db::ltm_ir::{ + OccurrenceAxis, OccurrenceRef, OccurrenceSite, OtherDepVerdict, derive_other_dep_verdict, +}; + +/// The ceteris-paribus wrap's view of the occurrence IR for ONE slot of a +/// target equation: the per-occurrence access shape and per-axis +/// classification `db::ltm_ir` already decided on the target's `Expr2` AST, +/// keyed by the structural child-index path with the slot prefix stripped. +/// +/// This is the SINGLE classifier family the wrap consumes. The wrap runs on +/// the target's printed-and-reparsed `Expr0`; rather than re-deriving each +/// occurrence's shape on that `Expr0` (the retired Expr0 mirror classifiers), +/// it tracks the same left-to-right child-index path `db::ltm_ir::walk_all_in_expr` +/// builds and looks the occurrence up here. The print->reparse round trip is +/// child-index-isomorphic to the `Expr2` walk (proved corpus-wide by +/// `classifier_agreement_tests::assert_occurrence_stream_aligns`), so a path +/// hit returns exactly the shape/axes the edge emitter used -- the two +/// families cannot drift because there is only one. +/// +/// A path MISS on a live-source subscript is the one residual production hazard +/// -- a novel shape the alignment gate cannot cover could make the reparse +/// non-isomorphic. That is NOT silently tolerated: the subscript arm flags it +/// (`WrapOutcome::missing_occurrence`) on a non-empty stream, so the partial is +/// abandoned with a loud skip-and-warn instead of freezing the live reference +/// into a constant-0 score. +/// +/// `for_slot` rebases to slot-local paths because the wrap walks a single +/// slot's expression from its root: an `Ast::Scalar`/`ApplyToAll` target is +/// slot `0`; an `Ast::Arrayed` target's per-element slots are numbered in +/// canonical element-key-sorted order (`build_arrayed_link_score_equation` +/// wraps each slot separately), matching the SiteId slot prefix. +pub(crate) struct OccurrenceLookup<'a> { + /// `(slot-local path, occurrence)` for every occurrence in this slot, in + /// document order. LTM equations are short, so a linear scan is cheaper + /// than hashing and keeps the borrow lifetimes trivial. + entries: Vec<(&'a [u16], &'a OccurrenceSite)>, +} + +impl<'a> OccurrenceLookup<'a> { + /// Build the lookup for `slot` of `occs` (the target's whole occurrence + /// stream), rebasing each occurrence's SiteId to its slot-local path. + pub(crate) fn for_slot(occs: &'a [OccurrenceSite], slot: u16) -> Self { + let entries = occs + .iter() + .filter(|o| o.site_id.0.first() == Some(&slot)) + .map(|o| (&o.site_id.0[1..], o)) + .collect(); + OccurrenceLookup { entries } + } + + /// The occurrence at exactly `path`, if any (a genuine causal reference at + /// that node). `None` for a node that is not a recorded occurrence (a + /// function name, a dimension name, a literal element selector, or a + /// deeper index the walk skipped). + fn get(&self, path: &[u16]) -> Option<&'a OccurrenceSite> { + self.entries + .iter() + .find(|(p, _)| *p == path) + .map(|(_, o)| *o) + } + + /// Whether this slot recorded NO occurrences. `true` for a genuinely + /// source-free slot -- an EXCEPT default that never references `from`, whose + /// trivial-zero guard form is legitimate, or an AGG-source generator whose + /// live source (the synthetic agg) is never a recorded occurrence. A miss on + /// a NON-empty lookup, by contrast, is a walker desync -- the `missing_occurrence` + /// guard in the subscript arm keys on this so it never false-fires on a + /// legitimately source-free slot. + fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Does any occurrence of the live source (a model `Variable` or a + /// `ModuleOutput` composite) STRICTLY under `prefix` carry access shape + /// `shape`? This is the occurrence-IR form of the retired + /// `expr0_contains_live_match` lookahead: an array-reducer `App` at + /// `prefix` is frozen whole (GH #517) unless it genuinely holds the live + /// reference, which is exactly "a live-shaped occurrence lives in its + /// subtree". + /// + /// Index-nested occurrences (`other_arr[live]`) are EXCLUDED: the retired + /// `expr0_contains_live_match` only inspected subscript *heads*, never + /// recursing into a subscript's index expressions, so an index-nested + /// `live` never made the enclosing reducer "hold the live ref". The + /// occurrence IR marks those `index_nested`, so filtering on it reproduces + /// that boundary exactly (the reducer freezes whole and the bare `live` + /// stays live -- `char_reducer_index_nested_freeze`). + /// + /// A `live` source can be either a model `Variable` (the ordinary link + /// score) or a `module·port` composite (`OccurrenceRef::ModuleOutput`, the + /// `db::module_link_score_equation` live channel). The retired + /// `expr0_contains_live_match` matched on the bare-`Var` ident text and so + /// treated BOTH the same -- a composite read bare inside a reducer made the + /// reducer hold the live ref. Matching on the occurrence's resolved name + /// (whichever variant) reproduces that: a module-output live source inside a + /// reducer recurses just like a variable one, instead of freezing whole. + fn subtree_has_live_shape( + &self, + prefix: &[u16], + live: &Ident, + shape: &RefShape, + ) -> bool { + self.entries.iter().any(|(p, o)| { + p.len() > prefix.len() + && p.starts_with(prefix) + && !o.index_nested + && &o.shape == shape + && occurrence_names_source(&o.reference, live) + }) + } +} + +/// Whether occurrence reference `reference` names the live source `live` -- +/// either a model `Variable` or a `module·port` composite +/// (`OccurrenceRef::ModuleOutput`). The wrap treats both alike (a bare read of +/// either is a live match), so the occurrence-IR predicates must too. +fn occurrence_names_source(reference: &OccurrenceRef, live: &Ident) -> bool { + match reference { + OccurrenceRef::Variable(v) => &Ident::::new(v) == live, + OccurrenceRef::ModuleOutput { composite, .. } => { + &Ident::::new(composite) == live + } + } +} /// The implicit WITH-LOOKUP rules (GH #910), in their own file only to keep this /// one under the project line-count lint. Re-exported below so callers keep @@ -65,10 +188,10 @@ pub(crate) struct IteratedDimCtx<'a> { pub dim_ctx: Option<&'a crate::dimensions::DimensionsContext>, /// Declared dimensions of the target's NON-LIVE array deps, keyed by /// canonical dep name (GH #526). Threaded by the db-bearing per-shape - /// link-score path so [`classify_other_dep_iterated_dim_subscript`] - /// can require position-and-mapping correspondence before collapsing - /// an iterated-dim subscript on a non-live dep to a bare - /// `PREVIOUS(dep)`. `None` (or a dep absent from the map -- an + /// link-score path so the other-dep verdict ([`other_dep_verdict`], via the + /// occurrence IR's per-axis classification) can require position-and-mapping + /// correspondence before collapsing an iterated-dim subscript on a non-live + /// dep to a bare `PREVIOUS(dep)`. `None` (or a dep absent from the map -- an /// implicit/synthetic name with no resolvable declaration) keeps the /// historical permissive collapse: declaring an unresolvable dep a /// mismatch would loud-skip edges that are correct today. @@ -103,6 +226,12 @@ pub(crate) struct IteratedDimCtx<'a> { /// `live_source` reference inside an apply-to-all-over-the-target's-dims /// equation pick the same element (the bare ref broadcasts/iterates that /// dimension). +/// +/// `#[cfg(test)]`: production reads the shape straight off the occurrence IR +/// (an iterated-dim subscript is classified `Bare` there); this Expr0 sibling +/// survives only for the wrap unit tests' occurrence builder and the alignment +/// gate. +#[cfg(test)] fn is_live_source_iterated_dim_subscript( indices: &[IndexExpr0], source_dim_elements: &[Vec], @@ -167,6 +296,32 @@ fn expr0_iterated_axis_lines_up( /// collapse an axis, and the `Wildcard` precedence check already ran). /// `None` when any index is neither -- the caller falls through to the /// legacy literal pass / `DynamicIndex`. +/// +/// Unlike the other Expr0 classifiers (now `#[cfg(test)]` since the wrap reads +/// the occurrence IR), this and [`expr0_iterated_axis_lines_up`] survive in +/// PRODUCTION behind a documented stage-3 seam. They run in the post-transform +/// row-pinning lowering ([`post_transform::rewrite_per_element_source_refs`]), +/// which re-classifies the ALREADY-WRAPPED `Expr0` -- a tree the wrap mutated by +/// inserting `PREVIOUS(...)` nodes, so its child-index structure no longer +/// matches the target's ORIGINAL `Expr2` and a `SiteId` computed on that +/// original AST cannot address it. The occurrence IR therefore cannot drive the +/// lowering; retiring these two requires stage 3 (delete the print->reparse +/// round trip so the lowering shares the IR's classification directly). See the +/// `ltm_augment_post_transform` module doc. +/// +/// Wrap-time occurrence TAGGING (attaching each occurrence's classification to +/// the node as the wrap builds it, so the post-transform pass reads a tag +/// instead of re-classifying) was rejected as the seam bridge: the tags the +/// lowering needs sit on source references INSIDE the `PREVIOUS(...)`/`INIT(...)` +/// subtrees the wrap froze, i.e. content the wrap descends past but never +/// re-enters, so a tag-only descent would have to walk exactly the pre-existing +/// PREVIOUS/INIT nodes the single-sourced wrap deliberately does not visit -- +/// resurrecting a second traversal of the same tree. The alternative, a +/// `Loc`-keyed side map from node location to occurrence, is itself a parallel +/// map keyed on a coordinate the wrap rewrites (a spliced `PREVIOUS` node +/// carries no original `Loc`), so it reintroduces exactly the drift surface this +/// track exists to remove. Stage 3's print->reparse deletion, by contrast, +/// makes the lowering read the ONE occurrence IR directly with no second map. fn classify_expr0_per_element_axes( indices: &[IndexExpr0], source_dim_elements: &[Vec], @@ -213,108 +368,62 @@ fn classify_expr0_per_element_axes( Some(axes) } -/// Classify an iterated-dimension subscript on a *non-live-source* -/// dependency (e.g. `pop[Region,Age]` in `growth[Region,Age] = -/// row_sum[Region] * c * pop[Region,Age]` while building the partial for -/// `(row_sum, growth)`). On [`OtherDepVerdict::Collapse`], -/// `wrap_non_matching_in_previous` collapses the subscript to a bare -/// `Var(dep)` before wrapping it in `PREVIOUS()` -- avoiding the -/// `PREVIOUS(Subscript(...))` codegen assertion. -/// -/// The firing precondition is unchanged from the pre-GH#526 recognizer: -/// every index is a bare `Var` naming one of the target's iterated -/// dimensions, with at most as many indices as the target has iterated -/// dimensions (anything else is [`OtherDepVerdict::NotIterated`]). -/// -/// What the verdict adds (GH #526) is the position-and-mapping -/// correspondence check against the dep's DECLARED dimensions -/// ([`IteratedDimCtx::dep_dims`], threaded by the db-bearing per-shape -/// path). For the *natural* case -- index `i` names (or positionally maps -/// to) the dep's `i`-th declared dimension -- each `dep[d_0, d_1, ...]` -/// in slot `(e_0, e_1, ...)` reads element `(e_0, e_1, ...)`, the same -/// element the bare `PREVIOUS(dep)` freezes, so the collapse is exact: -/// `Collapse`. A *transposed* reference (`arr[D2,D1]` for `arr` declared -/// `[D1,D2]`) is a genuine positional transposition in the executed -/// simulation -- in slot `(i, j)` it reads element `(j, i)` -- so the -/// collapse would freeze `arr[i,j]_prev` instead of `arr[j,i]_prev`, a -/// silent magnitude error in the link score: `Mismatch`, never collapsed. -/// The same holds for an arity mismatch and for a mapped pair without a -/// usable positional correspondence (the -/// [`crate::ltm_agg::iterated_axis_slot_elements`] gate, shared with the -/// live-source recognizer's mapped arm). -/// -/// A dep whose declared dims are UN-THREADABLE -- `dep_dims` is `None` -/// (callers without db access) or the dep is absent from the map (an -/// implicit/synthetic name) -- keeps the historical permissive -/// `Collapse`: declaring it a mismatch would loud-skip edges that are -/// correct today. (NOT the GH #762 reducer-body work, which covers the -/// source->agg per-row partials in `generate_nonlinear_body_partial`; -/// this is the other-dep iterated-subscript collapse in target-equation -/// partials.) -/// -/// Track A3 stage 2 status: the verdict RULE is single-sourced on -/// [`derive_other_dep_verdict`] (`db::ltm_ir`) -- the same rule the occurrence -/// IR's `axes` feed -- so the verdict cannot drift whichever family builds -/// `axes` (that rustdoc proves the shared arity guard dominates the sole -/// per-axis labeling difference; `verdict_ignores_over_arity_axis_labeling` -/// pins it). What is NOT yet single-sourced is the per-axis [`OccurrenceAxis`] -/// construction below -- still a textual mirror of `classify_occurrence_axes`; -/// retiring it needs the SiteId occurrence bridge into the wrap (deferred -/// stage-2 work; see the stage-2 report for why the wrap-surface + per-element -/// lowering conversion is multi-round). -fn classify_other_dep_iterated_dim_subscript( +/// Build the per-axis [`OccurrenceAxis`] classification of an iterated-dimension +/// subscript on a *non-live-source* dependency (e.g. `pop[Region,Age]` in +/// `growth[Region,Age] = row_sum[Region] * c * pop[Region,Age]`), the way +/// `db::ltm_ir::classify_occurrence_axes` classifies it on the `Expr2` AST. +/// +/// This is the Expr0 sibling that lets the `#[cfg(test)]` occurrence builder +/// ([`build_wrap_test_occurrences`]) reconstruct an occurrence's `axes` for the +/// wrap unit tests. PRODUCTION reads the axes straight off the occurrence IR +/// ([`OccurrenceLookup`]) and feeds them to the single-sourced +/// [`derive_other_dep_verdict`] via [`other_dep_verdict`] -- there is no Expr0 +/// re-derivation of the verdict on the live path, so the two families cannot +/// drift. +/// +/// Each index that names (or positionally maps to) the dep's declared axis at +/// that position is `Iterated`, a target-iterated name that does NOT line up is +/// `MismatchedIterated` (the GH #526 case a bare collapse must not silently +/// freeze), an over-dep-arity target-iterated name is `Iterated{d,d}` (the +/// verdict's arity guard dominates it), and any non-target-iterated-Var index is +/// `Dynamic` (which makes `derive_other_dep_verdict` return `NotIterated`, +/// matching the pre-flip recognizer's short-circuit). +#[cfg(test)] +fn other_dep_occurrence_axes( dep: &Ident, indices: &[IndexExpr0], ctx: Option<&IteratedDimCtx<'_>>, -) -> OtherDepVerdict { +) -> Vec { let Some(ctx) = ctx else { - return OtherDepVerdict::NotIterated; + return Vec::new(); }; - // Short-circuit before touching `dep_dims`: a non-empty subscript with no - // more indices than the target has iterated dims. (`derive_other_dep_verdict` - // re-checks these, but building the axes below already assumes them.) - if indices.is_empty() || indices.len() > ctx.target_iterated_dims.len() { - return OtherDepVerdict::NotIterated; - } let dep_dims = ctx.dep_dims.and_then(|m| m.get(dep.as_str())); - // Build the per-axis classification the promoted verdict rule consumes: an - // index lined up (by name or positional mapping) with the dep's declared - // axis at that position is `Iterated`, else `MismatchedIterated`. An index - // that is NOT a bare target-iterated-dim `Var` makes the whole subscript - // `NotIterated`. When the dep is un-threadable (`dep_dims` is `None`) the - // verdict's `None`-arity arm collapses regardless of the per-axis marking, - // and a position with no declared dep axis (an over-dep-arity index) is - // caught by the verdict's arity check first -- so marking those `Iterated` - // is sound (it only needs to keep every axis iterated/mismatched). - let mut axes: Vec = Vec::with_capacity(indices.len()); - for (i, idx) in indices.iter().enumerate() { - let IndexExpr0::Expr(Expr0::Var(name, _)) = idx else { - return OtherDepVerdict::NotIterated; - }; - let d = canonicalize(name.as_str()).into_owned(); - if !ctx.target_iterated_dims.iter().any(|t| t == &d) { - return OtherDepVerdict::NotIterated; - } - let axis = match dep_dims.and_then(|dd| dd.get(i)) { - Some(dep_dim) if other_dep_axis_lines_up(&d, dep_dim, ctx) => { - OccurrenceAxis::Iterated { - dim: d, - source_dim: canonicalize(dep_dim.name()).into_owned(), + indices + .iter() + .enumerate() + .map(|(i, idx)| { + let IndexExpr0::Expr(Expr0::Var(name, _)) = idx else { + return OccurrenceAxis::Dynamic; + }; + let d = canonicalize(name.as_str()).into_owned(); + if !ctx.target_iterated_dims.iter().any(|t| t == &d) { + return OccurrenceAxis::Dynamic; + } + match dep_dims.and_then(|dd| dd.get(i)) { + Some(dep_dim) if other_dep_axis_lines_up(&d, dep_dim, ctx) => { + OccurrenceAxis::Iterated { + dim: d, + source_dim: canonicalize(dep_dim.name()).into_owned(), + } } + Some(_) => OccurrenceAxis::MismatchedIterated { dim: d }, + None => OccurrenceAxis::Iterated { + dim: d.clone(), + source_dim: d, + }, } - Some(_) => OccurrenceAxis::MismatchedIterated { dim: d }, - None => OccurrenceAxis::Iterated { - dim: d.clone(), - source_dim: d, - }, - }; - axes.push(axis); - } - derive_other_dep_verdict( - &axes, - dep_dims.map(|dd| dd.len()), - ctx.target_iterated_dims.len(), - ) + }) + .collect() } /// Does iterated-dimension index `d` (canonical) line up with a non-live @@ -325,6 +434,7 @@ fn classify_other_dep_iterated_dim_subscript( /// `mapped_element_correspondence` gate (both declaration directions, /// positional mappings only) so the live-source and other-dep recognizers /// can never disagree about which mapped pairs are usable. +#[cfg(test)] fn other_dep_axis_lines_up( d: &str, dep_dim: &crate::dimensions::Dimension, @@ -344,8 +454,10 @@ fn other_dep_axis_lines_up( /// Classify an `Expr0` subscript's shape based on its indices. /// /// Mirrors `db::ltm_ir::resolve_literal_index`'s classification logic but at -/// the `Expr0` (parsed-AST) level — used by `wrap_non_matching_in_previous` -/// before subscripts have been lowered to `Expr2`. Each input string in +/// the `Expr0` (parsed-AST) level. `#[cfg(test)]`: the ceteris-paribus wrap no +/// longer re-derives shape on `Expr0` (it reads the occurrence IR); this sibling +/// survives only for the wrap unit tests' occurrence builder and the alignment +/// gate -- see [`resolve_literal_element_index`]. Each input string in /// `source_dim_elements` is the canonical lowercase element name for the /// corresponding source dimension, in source-declared order. /// @@ -374,6 +486,9 @@ fn other_dep_axis_lines_up( /// `Var` names are matched against the dimension at that position first /// and then against any dimension as a fallback (mirroring /// `classify_expr0_subscript_shape`'s match rules). +/// +/// `#[cfg(test)]`: see [`resolve_literal_element_index`]. +#[cfg(test)] fn is_literal_element_index( idx: &IndexExpr0, position: usize, @@ -386,22 +501,16 @@ fn is_literal_element_index( /// `db::ltm_ir::resolve_literal_index` (the Expr2 sibling) so both /// classifiers agree on what counts as a "literal element". /// -/// Two-family invariant (the tracked Track A3 stage-2/3 surface): the edge -/// emitter classifies each reference on the `Expr2` AST (`db::ltm_ir`), while -/// the ceteris-paribus wrap re-derives the shape on the printed-and-reparsed -/// `Expr0` via this sibling. If they disagree (e.g. on an out-of-range integer -/// literal -- `Expr2` -> `DynamicIndex`, `Expr0` -> `FixedIndex(...)`), the -/// shape comparison in `wrap_non_matching_in_previous` fails, every occurrence -/// of the source is `PREVIOUS`-wrapped, and the link score silently zeroes. -/// The A1 differential gate (`ltm_classifier_agreement_tests`) turns any drift -/// on a corpus-covered shape into a LOUD test failure; what it cannot cover is -/// a novel PRODUCTION shape, which is exactly why stage 2 deletes this Expr0 -/// sibling -- routing the shape/literal decision through the `db::ltm_ir` -/// occurrence stream (the SiteId bridge) makes the drift structurally -/// impossible for EVERY shape, not just the covered ones. (The other-dep -/// VERDICT is already single-sourced on `db::ltm_ir::derive_other_dep_verdict`; -/// the residual per-element lowering blocker is documented in the stage-2 -/// report.) +/// `#[cfg(test)]`: this Expr0 classifier no longer drives production. The +/// ceteris-paribus wrap consumes the occurrence IR ([`OccurrenceLookup`]) -- the +/// same `db::ltm_ir` classification the edge emitter uses -- so there is ONE +/// classifier family and the historical Expr0/Expr2 drift (which silently zeroed +/// a link score when the wrap re-derived a different shape than the emitter, e.g. +/// GH #759 / GH #913 / the `pop[01]` canonicalization) is structurally +/// impossible. This sibling survives only for the wrap unit tests' occurrence +/// builder ([`build_wrap_test_occurrences`]) and the alignment gate +/// (`ltm_classifier_agreement_tests`), which proves the reparsed-Expr0 walk and +/// the IR's `Expr2` walk stay path- and shape-isomorphic corpus-wide. /// /// Element names appear as `Var` nodes; integer literals appear as /// `Const` nodes whose text is the integer. Either form is validated @@ -410,6 +519,7 @@ fn is_literal_element_index( /// `Const("999", ...)` over an indexed dim of size 5 won't match and /// falls through to `None`. Matching prefers the dim at the index's /// position, falling back to any dim if not found there. +#[cfg(test)] fn resolve_literal_element_index( idx: &IndexExpr0, position: usize, @@ -447,6 +557,7 @@ fn resolve_literal_element_index( } } +#[cfg(test)] fn classify_expr0_subscript_shape( indices: &[IndexExpr0], source_dim_elements: &[Vec], @@ -520,83 +631,6 @@ fn is_array_reducer_name(name: &str, arity: usize) -> bool { crate::ltm_agg::reducer_kind_from_name(&name.to_ascii_lowercase(), arity).is_some() } -/// Does `expr` contain the live source reference the partial isolates -- a -/// bare `Var(live_source)` (when `live_shape` is `Bare`) or a -/// `Subscript(live_source, indices)` whose access shape equals `live_shape`? -/// -/// Used by [`wrap_non_matching_in_previous`] to decide whether an enclosing -/// array-reducer App is "other content" (so the whole reducer should be -/// `PREVIOUS`-wrapped -- `PREVIOUS(SUM(arr[*]))`, which evaluates fine) or -/// genuinely holds the live reference (so it must be recursed into, e.g. the -/// test-only `RefShape::Wildcard` path where `SUM(arr[*])` *is* the live -/// thing). See GH #517: the alternative -- recursing and emitting -/// `SUM(PREVIOUS(arr[*]))` -- is silently `0.0` at every step under an -/// active apply-to-all dimension because codegen has no -/// LoadPrev-of-array-view path. -fn expr0_contains_live_match( - expr: &Expr0, - live_source: &Ident, - live_shape: &RefShape, - source_dim_elements: &[Vec], - iter_ctx: Option<&IteratedDimCtx<'_>>, -) -> bool { - match expr { - Expr0::Const(..) => false, - Expr0::Var(ident, _) => { - matches!(live_shape, RefShape::Bare) && &Ident::new(ident.as_str()) == live_source - } - Expr0::Subscript(ident, indices, _) => { - // A `live_source` occurrence reachable only through an index - // expression (`other_arr[live_source]`) is never the captured - // live ref -- `wrap_index_non_matching_in_previous` passes a - // throwaway sink for those -- so we only consider a subscript - // whose *head* is `live_source` with the matching shape. An - // iterated-dimension subscript classifies `Bare` here too (it is - // collapsed to its head `Var` in `wrap_non_matching_in_previous`). - &Ident::new(ident.as_str()) == live_source - && &classify_expr0_subscript_shape(indices, source_dim_elements, iter_ctx) - == live_shape - } - Expr0::App(UntypedBuiltinFn(_, args), _) => args.iter().any(|a| { - expr0_contains_live_match(a, live_source, live_shape, source_dim_elements, iter_ctx) - }), - Expr0::Op1(_, inner, _) => expr0_contains_live_match( - inner, - live_source, - live_shape, - source_dim_elements, - iter_ctx, - ), - Expr0::Op2(_, l, r, _) => { - expr0_contains_live_match(l, live_source, live_shape, source_dim_elements, iter_ctx) - || expr0_contains_live_match( - r, - live_source, - live_shape, - source_dim_elements, - iter_ctx, - ) - } - Expr0::If(c, t, e, _) => { - expr0_contains_live_match(c, live_source, live_shape, source_dim_elements, iter_ctx) - || expr0_contains_live_match( - t, - live_source, - live_shape, - source_dim_elements, - iter_ctx, - ) - || expr0_contains_live_match( - e, - live_source, - live_shape, - source_dim_elements, - iter_ctx, - ) - } - } -} - /// Whether any subexpression of `expr` prints exactly as `reducer_text` -- the /// [`WrapCtx::live_reducer_text`] containment test (Track A stage 1, finding 2). /// @@ -660,6 +694,22 @@ struct WrapOutcome { /// verbatim), and `build_partial_equation_shaped_with_live_ref` /// returns the loud `UnfreezablePartial` error. other_dep_mismatch: bool, + /// Set when a live-source subscript node had NO occurrence at its tracked + /// structural path even though the slot's occurrence stream is non-empty -- + /// a walker desync (the reparsed-`Expr0` walk drifted from the IR's `Expr2` + /// `SiteId` numbering, which `assert_occurrence_stream_aligns` proves cannot + /// happen on the covered corpus but a NOVEL production shape might). On a + /// miss the shape lookup returns `None`, `node_shape == Some(live_shape)` + /// fails, and the live reference would be silently FROZEN -- a + /// `PREVIOUS(...)` that compiles cleanly and zeroes the score. That silent + /// zero is exactly the failure the single-classifier flip is meant to + /// eliminate, so the miss is surfaced LOUDLY instead: callers abandon the + /// partial (`shaped_guard_form_text` returns `Err` without even trying the + /// changed-last dual, which reads the SAME desynced stream, and + /// `build_partial_equation_shaped_with_live_ref` returns + /// `UnfreezablePartial`), and the db-bearing emitters turn that into a + /// skip-and-warn. + missing_occurrence: bool, } /// The immutable parameters of the ceteris-paribus wrap @@ -688,12 +738,29 @@ struct WrapCtx<'a> { /// Canonical idents of the non-`live_source` deps that must be wrapped; /// names outside this set (and not `live_source`) are left alone. other_deps: &'a HashSet>, - /// Per-source-axis element-name lists, in source-declared order. - source_dim_elements: &'a [Vec], /// GH #511 iterated-dimension context (`None` for a scalar live source). iter_ctx: Option<&'a IteratedDimCtx<'a>>, /// Project dims context for [`qualify_element_index`] (GH #587). dims_ctx: Option<&'a crate::dimensions::DimensionsContext>, + /// The occurrence IR for the slot being wrapped -- the single classifier + /// family. Every per-occurrence access-shape / iterated-collapse / + /// other-dep-verdict decision the wrap makes is a lookup here by the + /// occurrence's structural path (tracked as the wrap descends), not a + /// re-derivation on the reparsed `Expr0`. + occ: &'a OccurrenceLookup<'a>, +} + +/// Append `i` to `path`, yielding the child node's structural path. The wrap's +/// recursion mirrors `db::ltm_ir::walk_all_in_expr`'s child-index construction +/// exactly, so the path at any node equals that occurrence's `SiteId` (minus +/// the slot prefix, which [`OccurrenceLookup::for_slot`] already stripped) -- +/// the invariant `classifier_agreement_tests::assert_occurrence_stream_aligns` +/// proves corpus-wide. Cloning per descent is cheap: LTM equations are short. +fn child_path(path: &[u16], i: u16) -> Vec { + let mut v = Vec::with_capacity(path.len() + 1); + v.extend_from_slice(path); + v.push(i); + v } /// Walk an `Expr0` tree and wrap variable references in `PREVIOUS()` except @@ -741,16 +808,43 @@ struct WrapCtx<'a> { /// they were causal references (GH #587). `None` (test-only callers, or /// paths without project dims in scope) disables qualification, keeping the /// conservative wrapping behavior. -fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapOutcome) -> Expr0 { +/// The `Collapse` / `Mismatch` / `NotIterated` verdict for an iterated-dimension +/// subscript on a NON-live-source dependency, derived from the occurrence IR's +/// per-axis classification (`node_occ.axes`) plus the two arity facts the +/// occurrence does not carry: the dep's declared arity (from `iter_ctx.dep_dims`) +/// and the target's iterated-dim count. Delegates to the single-sourced +/// [`derive_other_dep_verdict`] -- the SAME rule `db::ltm_ir` feeds the edge +/// emitter -- so the wrap and the emitter cannot disagree on the verdict. +/// +/// `None` (no recorded occurrence, or no `iter_ctx` -- the scalar/agg callers +/// with no iterated-dimension space) yields `NotIterated`: there is no iterated +/// collapse to perform. +fn other_dep_verdict( + node_occ: Option<&OccurrenceSite>, + dep: &str, + iter_ctx: Option<&IteratedDimCtx<'_>>, +) -> OtherDepVerdict { + let (Some(occ), Some(ic)) = (node_occ, iter_ctx) else { + return OtherDepVerdict::NotIterated; + }; + let dep_arity = ic.dep_dims.and_then(|m| m.get(dep)).map(|d| d.len()); + derive_other_dep_verdict(&occ.axes, dep_arity, ic.target_iterated_dims.len()) +} + +fn wrap_non_matching_in_previous( + expr: Expr0, + ctx: &WrapCtx<'_>, + out: &mut WrapOutcome, + path: &[u16], +) -> Expr0 { // `dims_ctx` is consumed only by `wrap_index_non_matching_in_previous` (via - // `ctx`), so it is intentionally not bound here. + // `ctx`); `source_dim_elements` / `iter_ctx` are no longer read here (the + // occurrence IR carries the shape) so they are not bound. let &WrapCtx { live_source, live_shape, live_reducer_text, other_deps, - source_dim_elements, - iter_ctx, .. } = ctx; // Track A stage 1: hold a designated hoisted reducer subexpression LIVE @@ -791,78 +885,76 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO } Expr0::Subscript(ident, indices, loc) => { let canonical = Ident::new(ident.as_str()); - // GH #511: an iterated-dimension subscript reads the same element - // a bare reference would in each slot of the target equation, so - // normalize it to a bare `Var` here -- *before* the live/PREVIOUS - // dispatch -- and re-run the (much simpler) `Var` logic. The - // model equation is untouched; only the LTM partial's `Expr0` is - // rewritten, so simulation still evaluates `live_source[d_i]`. - // For the live source we use the precise position-matched check; - // for a non-live dep (`pop[Region,Age]` while building the - // `row_sum -> growth` partial) the verdict-based check, which - // collapses only on exact (or un-threadable-dims permissive) - // positional correspondence and flags a KNOWN mismatch instead - // of freezing the wrong element (GH #526). Either way, the - // alternative -- `PREVIOUS(Subscript(...))` -- trips the - // codegen assertion. + // The occurrence IR's classification of THIS subscript node -- the + // single classifier family. `db::ltm_ir` decided the access shape + // and per-axis reads on the target's `Expr2` AST; the wrap consults + // that here by the structural path it tracks (which equals the + // occurrence's `SiteId`, corpus-proven by + // `assert_occurrence_stream_aligns`) rather than re-deriving on the + // reparsed `Expr0`. A `None` means this node is not a recorded + // causal reference (so it is neither the live source nor an + // iterated other-dep the wrap collapses). + let node_occ = ctx.occ.get(path); + let node_shape = node_occ.map(|o| &o.shape); + // Loud desync guard (finding 1): the walker records an occurrence + // for EVERY live-source subscript head, so a `None` here on a + // NON-empty stream means the wrap's tracked path drifted from the + // IR's `SiteId` -- the shape lookup would silently miss, the live + // reference would freeze, and the score would zero. Flag it; the + // caller abandons the partial and warns rather than emitting the + // silent zero. (A genuinely source-free slot -- an EXCEPT default + // that never references `from`, or an agg-source generator whose + // synthetic-agg live source is never a walked node -- leaves the + // stream empty, so `is_empty()` excludes those legitimate misses.) + if &canonical == live_source && node_occ.is_none() && !ctx.occ.is_empty() { + out.missing_occurrence = true; + } + // GH #511: an iterated-dimension subscript on the LIVE source reads + // the same element a bare reference would in each slot, so + // `db::ltm_ir` classifies it `Bare` (`classify_iterated_dim_shape`, + // all axes `Iterated`; a plain subscript is never `Bare`). + // Normalize it to a bare `Var` -- *before* the live/PREVIOUS + // dispatch -- so `PREVIOUS(Var)` (which codegen accepts) replaces + // the `PREVIOUS(Subscript(...))` that trips the codegen assertion. + // + // Suppressed for a `PerElement` live shape: the per-element path + // (`generate_per_element_link_equation`) emits a SCALAR equation + // per (row, target element), so a live-source occurrence is frozen + // at `PREVIOUS` and then row-pinned AFTER the wrap + // (`rewrite_per_element_source_refs`) -- collapsing to bare here + // would discard the subscript that pin needs, leaving an over-arity + // `PREVIOUS(pop)` for a positionally-mapped occurrence that fails to + // compile and silently zeroes the score. Skip the collapse so the + // subscript survives; the post-transform pin resolves each iterated + // axis (mapped axes through the correspondence) to this element's + // own coordinate (byte-identical to bare for the unmapped case). if &canonical == live_source { - // The live-source iterated-dim collapse (`pop[Region,Age]` -> - // bare `Var(pop)`) is only sound when the emitted partial keeps - // the source reference bare-and-iterated -- the A2A / scalar - // partial paths, where a `Bare` live shape then reads the - // current element under the target's own A2A expansion. The - // `PerElement` live shape is the inverted per-element path - // (`generate_per_element_link_equation`): it emits a SCALAR - // equation per (row, target element), so any OTHER live-source - // occurrence in the body (an all-iterated `pop[Region,Age]` - // alongside the emitting `pop[Region,young]` site) is frozen at - // `PREVIOUS` and then pinned to its own concrete row AFTER the - // wrap (`rewrite_per_element_source_refs`, keyed on the - // subscript's per-axis classification). Collapsing to bare here - // discards the subscript that pin needs: the bare-`Var` pin - // projects by the source's OWN dims, so for a positionally- - // MAPPED occurrence (`pop[State,Age]`, State->Region) it finds - // no row in the target-keyed projection and leaves an over-arity - // `PREVIOUS(pop)` that fails to compile and silently zeroes the - // score. Skip the collapse so the subscript survives; the frozen - // occurrence keeps `pop[State,Age]` and the post-transform pin - // resolves each iterated axis -- mapped axes through the - // `State->Region` correspondence -- to this element's own - // coordinate. (For the unmapped case the pin resolves the same - // row either way, so the output is byte-identical.) if !matches!(live_shape, RefShape::PerElement { .. }) - && is_live_source_iterated_dim_subscript( - &indices, - source_dim_elements, - iter_ctx, - ) + && node_shape == Some(&RefShape::Bare) { - return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out); + return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out, path); } } else if other_deps.contains(&canonical) && !matches!(live_shape, RefShape::PerElement { .. }) { - // The other-dep iterated-subscript collapse (`w[Age]` -> bare - // `PREVIOUS(w)`) is only sound when the emitted partial keeps - // the dep bare-and-iterated -- the A2A / scalar partial paths, - // whose result is an `Equation::ApplyToAll` that reads the - // current element. The `PerElement` live shape is the inverted - // per-element path (`generate_per_element_link_equation`): it - // emits a SCALAR equation per (row, target element) and pins - // every arrayed dep to that element afterward - // (`subscript_idents_at_element`). Collapsing to bare there - // would let the full-target-tuple pin over-subscript a - // subset-dims dep (`w[region·boston, age·old]` -- arity 2 over - // a 1-D `w`), producing a fragment that fails to compile and - // silently zeroes the score. Skip the collapse so the dep - // keeps its subscript (`PREVIOUS(w[Age])`) and the pin resolves - // each dimension-name index to this element's own coordinate - // (`PREVIOUS(w[age·old])`). The GH #526 `Mismatch` doom cannot - // fire on this path anyway (its `dep_dims` are `None`), so - // skipping the whole verdict preserves behavior. - match classify_other_dep_iterated_dim_subscript(&canonical, &indices, iter_ctx) { + // The other-dep iterated-subscript verdict (`w[Age]` -> bare + // `PREVIOUS(w)` on `Collapse`), derived from the occurrence's + // per-axis classification via the single-sourced + // `derive_other_dep_verdict` -- the SAME rule the edge emitter's + // IR feeds, so the wrap and the emitter cannot disagree. Only + // sound when the emitted partial keeps the dep bare-and-iterated + // (the A2A / scalar paths); the `PerElement` guard above skips + // it (that path pins every arrayed dep per element afterward, so + // a bare collapse would let a full-target-tuple pin over-subscript + // a subset-dims dep -- an uncompilable, silently-zeroed fragment). + match other_dep_verdict(node_occ, canonical.as_str(), ctx.iter_ctx) { OtherDepVerdict::Collapse => { - return wrap_non_matching_in_previous(Expr0::Var(ident, loc), ctx, out); + return wrap_non_matching_in_previous( + Expr0::Var(ident, loc), + ctx, + out, + path, + ); } OtherDepVerdict::Mismatch => { // GH #526: collapsing would freeze the WRONG element. @@ -876,45 +968,42 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO OtherDepVerdict::NotIterated => {} } } - // Classify the subscript's shape using the ORIGINAL indices - // BEFORE recursing into them. If a user variable shares a - // name with a dimension element (e.g., a variable also named - // `NYC`), recursing first would rewrite `Var(NYC)` as - // `App(PREVIOUS, [Var(NYC)])`, and then classification would - // fall through to `DynamicIndex`, breaking a live FixedIndex - // shape match. - let subscript_shape = - classify_expr0_subscript_shape(&indices, source_dim_elements, iter_ctx); - if &canonical == live_source && &subscript_shape == live_shape { + if &canonical == live_source && node_shape == Some(live_shape) { // Live reference: the OUTER subscript stays unwrapped. // Decide per-index whether to recurse: // - // - Literal element refs (Var matching a dim element, - // or an integer literal in an indexed dim) are - // dimension references at runtime; leave them - // verbatim so a variable/element name collision - // doesn't wrap them. + // - A literal element SELECTOR (`db::ltm_ir` marks it + // `OccurrenceAxis::Pinned`, and the walk pushes no `SiteId` + // under it) is a runtime dimension reference; leave it + // verbatim so a variable/element name collision doesn't wrap + // it, and so the tracked path lines up with the walk. // - // - Wildcard tokens (`*`) have no inner content to - // wrap; recursing is a no-op so doing it for - // uniformity is fine. + // - Wildcard tokens (`*`) have no inner content to wrap; + // recursing is a no-op. // - // - Non-literal indices (expressions like `idx + helper` - // in `RefShape::DynamicIndex`) are computational - // content; recurse so any `other_deps` referenced - // inside get held at PREVIOUS for ceteris-paribus. + // - Non-literal indices (expressions like `idx + helper` in + // `RefShape::DynamicIndex`) are computational content; + // recurse so any `other_deps` referenced inside get held at + // PREVIOUS for ceteris-paribus. // - // Without the per-index split, DynamicIndex live refs - // would skip wrapping inner deps and the partial - // equation would no longer be ceteris-paribus. + // Without the per-index split, DynamicIndex live refs would skip + // wrapping inner deps and the partial would no longer be + // ceteris-paribus. + let occ_axes = node_occ.map(|o| o.axes.as_slice()).unwrap_or(&[]); let indices: Vec = indices .into_iter() .enumerate() .map(|(i, idx)| { - if is_literal_element_index(&idx, i, source_dim_elements) { + if matches!(occ_axes.get(i), Some(OccurrenceAxis::Pinned(_))) { idx } else { - wrap_index_non_matching_in_previous(idx, ctx, out, false) + wrap_index_non_matching_in_previous( + idx, + ctx, + out, + false, + &child_path(path, i as u16), + ) } }) .collect(); @@ -964,8 +1053,15 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO &canonical == live_source && matches!(live_shape, RefShape::PerElement { .. }); let indices: Vec = indices .into_iter() - .map(|idx| { - wrap_index_non_matching_in_previous(idx, ctx, out, skip_index_qualification) + .enumerate() + .map(|(i, idx)| { + wrap_index_non_matching_in_previous( + idx, + ctx, + out, + skip_index_qualification, + &child_path(path, i as u16), + ) }) .collect(); let subscript = Expr0::Subscript(ident, indices, loc); @@ -1005,10 +1101,20 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO "lookup" | "lookup_forward" | "lookup_backward" ) && !args.is_empty() { - let mut args_iter = args.into_iter(); - let table_arg = args_iter.next().expect("checked non-empty"); - let mut new_args = vec![table_arg]; - new_args.extend(args_iter.map(|a| wrap_non_matching_in_previous(a, ctx, out))); + // The table arg (child 0) is held verbatim; only the index + // arg(s) are wrapped. Child indices match the `Expr2` walk, + // which counts the skipped `LookupTable` slot as child 0. + let new_args = args + .into_iter() + .enumerate() + .map(|(i, a)| { + if i == 0 { + a + } else { + wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i as u16)) + } + }) + .collect(); return Expr0::App(UntypedBuiltinFn(name, new_args), loc); } // GH #517: an array-reducer subexpression (`SUM(pop[*])`, @@ -1037,40 +1143,67 @@ fn wrap_non_matching_in_previous(expr: Expr0, ctx: &WrapCtx<'_>, out: &mut WrapO .is_some_and(|text| args.iter().any(|a| expr0_contains_reducer_text(a, text))); if is_array_reducer_name(&name, args.len()) && !holds_live_reducer - && !args.iter().any(|a| { - expr0_contains_live_match( - a, - live_source, - live_shape, - source_dim_elements, - iter_ctx, - ) - }) + && !ctx + .occ + .subtree_has_live_shape(path, live_source, live_shape) { let reducer = Expr0::App(UntypedBuiltinFn(name, args), loc); return Expr0::App(UntypedBuiltinFn("PREVIOUS".to_string(), vec![reducer]), loc); } let args = args .into_iter() - .map(|a| wrap_non_matching_in_previous(a, ctx, out)) + .enumerate() + .map(|(i, a)| { + wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i as u16)) + }) .collect(); Expr0::App(UntypedBuiltinFn(name, args), loc) } Expr0::Op1(op, inner, loc) => Expr0::Op1( op, - Box::new(wrap_non_matching_in_previous(*inner, ctx, out)), + Box::new(wrap_non_matching_in_previous( + *inner, + ctx, + out, + &child_path(path, 0), + )), loc, ), Expr0::Op2(op, lhs, rhs, loc) => Expr0::Op2( op, - Box::new(wrap_non_matching_in_previous(*lhs, ctx, out)), - Box::new(wrap_non_matching_in_previous(*rhs, ctx, out)), + Box::new(wrap_non_matching_in_previous( + *lhs, + ctx, + out, + &child_path(path, 0), + )), + Box::new(wrap_non_matching_in_previous( + *rhs, + ctx, + out, + &child_path(path, 1), + )), loc, ), Expr0::If(cond, then_expr, else_expr, loc) => Expr0::If( - Box::new(wrap_non_matching_in_previous(*cond, ctx, out)), - Box::new(wrap_non_matching_in_previous(*then_expr, ctx, out)), - Box::new(wrap_non_matching_in_previous(*else_expr, ctx, out)), + Box::new(wrap_non_matching_in_previous( + *cond, + ctx, + out, + &child_path(path, 0), + )), + Box::new(wrap_non_matching_in_previous( + *then_expr, + ctx, + out, + &child_path(path, 1), + )), + Box::new(wrap_non_matching_in_previous( + *else_expr, + ctx, + out, + &child_path(path, 2), + )), loc, ), } @@ -1125,6 +1258,7 @@ fn wrap_index_non_matching_in_previous( ctx: &WrapCtx<'_>, out: &mut WrapOutcome, skip_element_qualification: bool, + path: &[u16], ) -> IndexExpr0 { // Only `dims_ctx` (element/dimension recognition) and `iter_ctx` (the // iterated-dim-name guard) are read directly here; the full wrap context @@ -1200,18 +1334,24 @@ fn wrap_index_non_matching_in_previous( // `other_dep_mismatch` doom DOES propagate: an index-nested mismatched // collapse dooms the changed-first partial just the same. let mut idx_out = WrapOutcome::default(); + // The index expression is at `path` (the walk pushes the index position + // before descending); a `Range`'s two operands are children 0 and 1 of + // that, mirroring `walk_all_in_expr`. let result = match index { IndexExpr0::Expr(e) => { - IndexExpr0::Expr(wrap_non_matching_in_previous(e, ctx, &mut idx_out)) + IndexExpr0::Expr(wrap_non_matching_in_previous(e, ctx, &mut idx_out, path)) } IndexExpr0::Range(l, r, loc) => IndexExpr0::Range( - wrap_non_matching_in_previous(l, ctx, &mut idx_out), - wrap_non_matching_in_previous(r, ctx, &mut idx_out), + wrap_non_matching_in_previous(l, ctx, &mut idx_out, &child_path(path, 0)), + wrap_non_matching_in_previous(r, ctx, &mut idx_out, &child_path(path, 1)), loc, ), other => other, }; out.other_dep_mismatch |= idx_out.other_dep_mismatch; + // A live-source subscript nested inside an index (`other[from[x]]`) desyncs + // the same way; propagate its loud miss flag out of the throwaway sink. + out.missing_occurrence |= idx_out.missing_occurrence; result } @@ -1392,22 +1532,41 @@ pub(crate) fn build_partial_equation_shaped_with_live_ref( iter_ctx: Option<&IteratedDimCtx<'_>>, dims_ctx: Option<&crate::dimensions::DimensionsContext>, ) -> Result<(String, Option), PartialEquationError> { + // Reconstruct the occurrence IR the production wrap consumes from the raw + // equation text (the production callers get it from `model_ltm_reference_sites`; + // this text-level test entry rebuilds an equivalent stream on the reparsed + // Expr0, using the `#[cfg(test)]` Expr0 classifiers the alignment gate proves + // stay in step with the IR). Slot-0 body, matching `wrap_changed_first_ast`. + let occurrences = build_wrap_test_occurrences( + equation_text, + live_source, + deps, + source_dim_elements, + iter_ctx, + ); + let occ = OccurrenceLookup::for_slot(&occurrences, 0); let (transformed, out) = wrap_changed_first_ast( equation_text, deps, live_source, live_shape, None, - source_dim_elements, iter_ctx, dims_ctx, + &occ, )?; - if out.other_dep_mismatch { + if out.other_dep_mismatch || out.missing_occurrence { return Err(PartialEquationError::unfreezable(equation_text)); } Ok((print_eqn(&transformed), out.live_ref)) } +#[cfg(test)] +#[path = "ltm_augment_wrap_test_support.rs"] +mod wrap_test_support; +#[cfg(test)] +pub(crate) use wrap_test_support::{build_wrap_test_occurrences, test_occurrences_for_var}; + /// The shared changed-first transform: filter `deps` down to the /// other-deps set, parse `equation_text`, and PREVIOUS-wrap via /// [`wrap_non_matching_in_previous`] -- returning the transformed AST (not @@ -1434,9 +1593,9 @@ fn wrap_changed_first_ast( live_source: &Ident, live_shape: &RefShape, live_reducer_text: Option<&str>, - source_dim_elements: &[Vec], iter_ctx: Option<&IteratedDimCtx<'_>>, dims_ctx: Option<&crate::dimensions::DimensionsContext>, + occ: &OccurrenceLookup<'_>, ) -> Result<(Expr0, WrapOutcome), PartialEquationError> { let other_deps: HashSet> = deps .iter() @@ -1453,12 +1612,15 @@ fn wrap_changed_first_ast( live_shape, live_reducer_text, other_deps: &other_deps, - source_dim_elements, iter_ctx, dims_ctx, + occ, }; let mut out = WrapOutcome::default(); - let transformed = wrap_non_matching_in_previous(ast, &ctx, &mut out); + // The wrap walks the slot's expression from its root; the occurrence + // lookup was already rebased to slot-local paths, so the root path is + // empty. + let transformed = wrap_non_matching_in_previous(ast, &ctx, &mut out, &[]); Ok((transformed, out)) } @@ -1672,9 +1834,9 @@ fn wrap_live_shaped_in_previous( expr: Expr0, live_source: &Ident, live_shape: &RefShape, - source_dim_elements: &[Vec], - iter_ctx: Option<&IteratedDimCtx<'_>>, frozen_ref: &mut Option, + occ: &OccurrenceLookup<'_>, + path: &[u16], ) -> Expr0 { match expr { Expr0::Const(..) => expr, @@ -1692,24 +1854,21 @@ fn wrap_live_shaped_in_previous( } Expr0::Subscript(ident, indices, loc) => { if &Ident::::new(ident.as_str()) == live_source { + let node_shape = occ.get(path).map(|o| &o.shape); // GH #511 normalization: an iterated-dim subscript reads the - // same element a bare reference would in each slot, and the - // IR classifies such a site `Bare`. - if matches!(live_shape, RefShape::Bare) - && is_live_source_iterated_dim_subscript( - &indices, - source_dim_elements, - iter_ctx, - ) - { + // same element a bare reference would in each slot, and + // `db::ltm_ir` classifies such a site `Bare` (all axes + // `Iterated`; a plain subscript is never `Bare`). This walker + // does not descend into subscript indices, so the head is the + // only occurrence looked up at `path`. + if matches!(live_shape, RefShape::Bare) && node_shape == Some(&RefShape::Bare) { let bare = Expr0::Var(ident, loc); if frozen_ref.is_none() { *frozen_ref = Some(bare.clone()); } return Expr0::App(UntypedBuiltinFn("PREVIOUS".to_string(), vec![bare]), loc); } - let shape = classify_expr0_subscript_shape(&indices, source_dim_elements, iter_ctx); - if &shape == live_shape { + if node_shape == Some(live_shape) { let subscript = Expr0::Subscript(ident, indices, loc); if frozen_ref.is_none() { *frozen_ref = Some(subscript.clone()); @@ -1730,14 +1889,15 @@ fn wrap_live_shaped_in_previous( } let args = args .into_iter() - .map(|a| { + .enumerate() + .map(|(i, a)| { wrap_live_shaped_in_previous( a, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, i as u16), ) }) .collect(); @@ -1749,9 +1909,9 @@ fn wrap_live_shaped_in_previous( *inner, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 0), )), loc, ), @@ -1761,17 +1921,17 @@ fn wrap_live_shaped_in_previous( *lhs, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 0), )), Box::new(wrap_live_shaped_in_previous( *rhs, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 1), )), loc, ), @@ -1780,25 +1940,25 @@ fn wrap_live_shaped_in_previous( *c, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 0), )), Box::new(wrap_live_shaped_in_previous( *t, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 1), )), Box::new(wrap_live_shaped_in_previous( *e, live_source, live_shape, - source_dim_elements, - iter_ctx, frozen_ref, + occ, + &child_path(path, 2), )), loc, ), @@ -1860,6 +2020,7 @@ fn shaped_guard_form_text( dims_ctx: Option<&crate::dimensions::DimensionsContext>, target_ref: &str, gf_table_ref: Option<&str>, + occ: &OccurrenceLookup<'_>, ) -> Result { let gf_wrap = |partial: String| -> String { match gf_table_ref { @@ -1873,10 +2034,17 @@ fn shaped_guard_form_text( from, shape, None, - source_dim_elements, iter_ctx, dims_ctx, + occ, )?; + // A walker desync (finding 1): the changed-first partial silently froze the + // live reference, and the changed-last dual would read the SAME desynced + // occurrence stream, so BOTH conventions are corrupt. Skip loudly rather + // than emit either silent zero. + if out.missing_occurrence { + return Err(PartialEquationError::unfreezable(equation_text)); + } if !out.other_dep_mismatch && !contains_unfreezable_previous(&changed_first) { let source_ref = source_ref_for_guard( from, @@ -1927,14 +2095,7 @@ fn shaped_guard_form_text( } let mut frozen_ref: Option = None; - let changed_last = wrap_live_shaped_in_previous( - ast, - from, - shape, - source_dim_elements, - iter_ctx, - &mut frozen_ref, - ); + let changed_last = wrap_live_shaped_in_previous(ast, from, shape, &mut frozen_ref, occ, &[]); let Some(frozen) = frozen_ref else { // No matching occurrence: the "frozen" equation would be the // target's own equation, scoring a silent constant 0. @@ -2516,6 +2677,18 @@ fn subscript_idents_in_expr0( /// equation, i.e. gf-INPUT units, while the guard form ratios it against /// gf-OUTPUT target deltas. `None` for an ordinary (gf-less) target. See /// [`WithLookupSlotRefs`]. +/// +/// `occ` is the target slot's occurrence-IR lookup the ceteris-paribus wrap +/// consumes -- the SINGLE classifier family every caller threads. A SCALAR-source +/// caller (`try_scalar_to_arrayed_link_scores`) NEEDS it: `from` is a model +/// `Variable` that can appear bare inside a reducer of the target's equation, and +/// the GH #517 reducer-freeze arm consults the stream to decide whether to freeze +/// that reducer whole or recurse into it. An empty stream froze it whole, +/// silently zeroing a score HEAD scored (or loudly declined). An AGG-source +/// caller (`from = $⁚ltm⁚agg⁚n`) also threads the real stream, but it is +/// behavior-neutral there: the live source is a synthetic aggregate held live by +/// `reducer_subst` text-matching, never a recorded occurrence, so every wrap +/// lookup for it misses whether the stream is empty or not. #[allow(clippy::too_many_arguments)] pub(crate) fn generate_scalar_to_element_equation( from: &str, @@ -2529,6 +2702,7 @@ pub(crate) fn generate_scalar_to_element_equation( source_pins: &[(Ident, String)], dims_ctx: Option<&crate::dimensions::DimensionsContext>, gf_table_ref: Option<&str>, + occ: &OccurrenceLookup<'_>, ) -> Result { let from_canonical = Ident::new(from); let from_q = quote_ident(from); @@ -2544,17 +2718,34 @@ pub(crate) fn generate_scalar_to_element_equation( // there is no iterated-dim context. The reducer -> agg-name substitution is // a POST-transform lowering of the wrapped AST, so the later element-pinning // / `source_pins` passes see the same agg-named text they did before. + // + // `occ` is the target slot's real occurrence stream (threaded by every + // caller); the wrap consults it at the reducer-freeze arm so a scalar `from` + // read bare inside a reducer recurses exactly like the shaped per-shape path + // does. For an agg `from` the stream is behavior-neutral (the agg is never a + // recorded occurrence), so the agg callers thread it too rather than a + // separate empty-stream path. let live_reducer_text = live_reducer_text_for_agg(reducer_subst, from); - let (wrapped, _out) = wrap_changed_first_ast( + let (wrapped, out) = wrap_changed_first_ast( to_elem_eqn_text, to_deps, &from_canonical, &RefShape::Bare, live_reducer_text, - &[], None, dims_ctx, + occ, )?; + // Loud degradation over a silent zero (matching `shaped_guard_form_text` / + // `build_partial_equation_shaped_with_live_ref`): a walker desync + // (`missing_occurrence`) or a GH #526 mismatched other-dep + // (`other_dep_mismatch`) makes the changed-first partial unusable. Both are + // inert for the agg-source callers (the agg is never a walked node, so + // `missing_occurrence` cannot fire, and `iter_ctx` is `None`), so this only + // affects the scalar-source path. + if out.missing_occurrence || out.other_dep_mismatch { + return Err(PartialEquationError::unfreezable(to_elem_eqn_text)); + } let partial = print_eqn(&substitute_reducers_in_expr0(wrapped, reducer_subst)); let mut partial = subscript_idents_at_element(&partial, to_deps_to_subscript, element)?; // Pin each mapped ident's references (the live source's numerator @@ -2626,6 +2817,7 @@ pub(crate) fn generate_per_element_link_equation( target_iterated_dims: &[String], dims_ctx: &crate::dimensions::DimensionsContext, gf_table_ref: Option<&str>, + occ: &OccurrenceLookup<'_>, ) -> Result { let from_canonical = Ident::::new(from); let source_dim_elements: Vec> = @@ -2677,9 +2869,9 @@ pub(crate) fn generate_per_element_link_equation( &from_canonical, &live_shape, None, - &source_dim_elements, Some(&iter_ctx), Some(dims_ctx), + occ, )?; if out.other_dep_mismatch { return Err(PartialEquationError::unfreezable(to_elem_eqn_text)); @@ -2733,6 +2925,16 @@ pub(crate) fn generate_per_element_link_equation( /// partial re-evaluates the gf's *input*, so it must be fed through the table /// to be commensurable with the `PREVIOUS(to)` anchor the guard form subtracts /// (GH #910). +/// +/// `occ` is the (scalar) target's occurrence stream. Its live source is the +/// synthetic agg (`$⁚ltm⁚agg⁚n`), which is NEVER a recorded occurrence -- it is +/// held live by `reducer_subst` text-matching, and during the wrap the reducer +/// that became it is still spelled `SUM(...)` -- so every `subtree_has_live_shape` +/// / `get` for the agg misses and the stream is behavior-neutral here (the +/// co-reducers freeze whole, the agg reducer is held live by text). The caller +/// threads the REAL stream regardless: there is one classifier family, not an +/// empty-stream shadow path. +#[allow(clippy::too_many_arguments)] // threads the agg-to-target generation context pub(crate) fn generate_agg_to_scalar_target_equation( agg_name: &str, to_name: &str, @@ -2741,6 +2943,7 @@ pub(crate) fn generate_agg_to_scalar_target_equation( to_deps: &HashSet>, dims_ctx: Option<&crate::dimensions::DimensionsContext>, gf_table_ref: Option<&str>, + occ: &OccurrenceLookup<'_>, ) -> Result { let agg_canonical = Ident::new(agg_name); let agg_q = quote_ident(agg_name); @@ -2757,9 +2960,9 @@ pub(crate) fn generate_agg_to_scalar_target_equation( &agg_canonical, &RefShape::Bare, live_reducer_text, - &[], None, dims_ctx, + occ, )?; let mut partial = print_eqn(&substitute_reducers_in_expr0(wrapped, reducer_subst)); if let Some(table_ref) = gf_table_ref { @@ -3200,6 +3403,7 @@ pub(crate) fn generate_link_score_equation_for_link( all_vars: &HashMap, Variable>, dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, + to_occurrences: &[OccurrenceSite], ) -> Result { generate_link_score_equation( from, @@ -3210,6 +3414,7 @@ pub(crate) fn generate_link_score_equation_for_link( all_vars, dim_ctx, dep_dims, + to_occurrences, ) } @@ -3230,6 +3435,7 @@ fn generate_link_score_equation( all_vars: &HashMap, Variable>, dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, + to_occurrences: &[OccurrenceSite], ) -> Result { // Check if this is a stock-to-flow link let is_stock_to_flow = matches!(all_vars.get(from), Some(Variable::Stock { .. })) @@ -3264,6 +3470,7 @@ fn generate_link_score_equation( to_var, dim_ctx, dep_dims, + to_occurrences, ) } else { // Use standard auxiliary-to-auxiliary formula @@ -3277,6 +3484,7 @@ fn generate_link_score_equation( to_var, dim_ctx, dep_dims, + to_occurrences, ) } } @@ -3417,6 +3625,7 @@ fn build_arrayed_link_score_equation( to_var: &Variable, dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, + to_occurrences: &[OccurrenceSite], ) -> Result { // The #511 iterated-dimension context for the per-slot partials: each // per-element slot can itself reference `from` by an iterated dimension @@ -3446,8 +3655,13 @@ fn build_arrayed_link_score_equation( .map(String::as_str) .chain(source_dim_names.iter().map(String::as_str)) .collect(); + // `slot` is the per-element occurrence-stream slot: `walk_all_in_expr` + // numbers `Ast::Arrayed` slots in canonical element-key-sorted order (then + // the default after the last element), so the wrap for element `slot` + // consumes exactly that slot's occurrences. let slot_equation = |expr: &crate::ast::Expr2, - gf_table_ref: Option<&str>| + gf_table_ref: Option<&str>, + slot: u16| -> Result { let elem_eqn_text = crate::patch::expr2_to_string(expr); // Per-element dependency set: walk *only this slot's* expression @@ -3465,6 +3679,7 @@ fn build_arrayed_link_score_equation( .into_iter() .filter(|d| !source_dim_token_set.contains(d.as_str())) .collect(); + let occ = OccurrenceLookup::for_slot(to_occurrences, slot); shaped_guard_form_text( &elem_eqn_text, &deps_e, @@ -3476,39 +3691,50 @@ fn build_arrayed_link_score_equation( dim_ctx, target_ref, gf_table_ref, + &occ, ) }; - // Sort the slots by element name so the resulting `Vec` -- which lands - // in a salsa-tracked `LtmVariablesResult` -- is deterministic - // regardless of `HashMap` iteration order across runs. + // Visit slots in canonical element-key-sorted order (matching + // `walk_all_in_expr`'s slot numbering), so both the emitted `Vec` and the + // per-slot occurrence lookup are deterministic and aligned. // The implicit WITH-LOOKUP slot wraps (GH #910): each element's own table, // the shared variable-level table, or no wrap for a gf-less element. // Resolved once for the whole target -- see `WithLookupSlotRefs`. let slot_refs = WithLookupSlotRefs::new(to_var, target_ast_dims); - let mut elements: Vec<( + let mut sorted_slots: Vec<(&crate::common::CanonicalElementName, &crate::ast::Expr2)> = + per_elem.iter().collect(); + sorted_slots.sort_by(|a, b| a.0.cmp(b.0)); + + let elements: Vec<( String, String, Option, Option, - )> = per_elem + )> = sorted_slots .iter() - .map(|(elem, expr)| { + .enumerate() + .map(|(slot, (elem, expr))| { let gf_table_ref = slot_refs.for_element(elem); Ok(( elem.as_str().to_string(), - slot_equation(expr, gf_table_ref.as_deref())?, + slot_equation(expr, gf_table_ref.as_deref(), slot as u16)?, None, None, )) }) .collect::>()?; - elements.sort_by(|a, b| a.0.cmp(&b.0)); let default_gf_table_ref = slot_refs.for_default(); let default_slot = default_expr - .map(|expr| slot_equation(expr, default_gf_table_ref.as_deref())) + .map(|expr| { + slot_equation( + expr, + default_gf_table_ref.as_deref(), + sorted_slots.len() as u16, + ) + }) .transpose()?; Ok(Equation::Arrayed( @@ -3627,6 +3853,7 @@ fn generate_auxiliary_to_auxiliary_equation( to_var: &Variable, dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, + to_occurrences: &[OccurrenceSite], ) -> Result { use crate::ast::Ast; @@ -3653,6 +3880,7 @@ fn generate_auxiliary_to_auxiliary_equation( to_var, dim_ctx, dep_dims, + to_occurrences, ); } @@ -3676,6 +3904,9 @@ fn generate_auxiliary_to_auxiliary_equation( dim_ctx, dep_dims, }; + // A scalar / `Ast::ApplyToAll` target is a single body -- slot 0 of the + // occurrence stream. + let occ = OccurrenceLookup::for_slot(to_occurrences, 0); let text = shaped_guard_form_text( &to_equation, &deps, @@ -3689,6 +3920,7 @@ fn generate_auxiliary_to_auxiliary_equation( // The implicit WITH-LOOKUP wrap for a tables-carrying target // (GH #910); `None` for an ordinary aux. with_lookup_table_ref(to_var).as_deref(), + &occ, )?; Ok(link_score_equation_for_target(text, to_var)) } @@ -3928,6 +4160,7 @@ fn generate_stock_to_flow_equation( flow_var: &Variable, dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, + to_occurrences: &[OccurrenceSite], ) -> Result { // For stock-to-flow, we need to calculate how the stock influences the flow // This is similar to auxiliary-to-auxiliary but we know the 'from' is a stock @@ -3953,6 +4186,7 @@ fn generate_stock_to_flow_equation( flow_var, dim_ctx, dep_dims, + to_occurrences, ); } @@ -3984,6 +4218,7 @@ fn generate_stock_to_flow_equation( // error -- see `source_ref_for_guard` (applied inside // `shaped_guard_form_text`, which also handles the GH #743 // changed-last fallback for an unfreezable changed-first partial). + let occ = OccurrenceLookup::for_slot(to_occurrences, 0); let text = shaped_guard_form_text( &flow_equation, &deps, @@ -3996,6 +4231,7 @@ fn generate_stock_to_flow_equation( target_ref, // A flow can be an implicit WITH-LOOKUP variable too (GH #910). with_lookup_table_ref(flow_var).as_deref(), + &occ, )?; Ok(link_score_equation_for_target(text, flow_var)) } diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index a01a8853b..7487ba43a 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -2892,13 +2892,14 @@ fn partial_equation_does_not_rewrap_inside_previous() { /// /// `to = SUM(w[from]) + from` with live source `from` (Bare): `from` occurs /// bare (outside the reducer -- the live occurrence) AND index-nested inside -/// `w[from]`. `expr0_contains_live_match`'s Subscript arm matches only a -/// subscript whose HEAD is the live source, so the index-nested `from` never -/// makes the reducer "hold the live reference": the whole reducer is frozen -/// (`PREVIOUS(sum(w[from]))`) and only the bare `from` stays live. +/// `w[from]`. The reducer-freeze check (`OccurrenceLookup::subtree_has_live_shape`) +/// EXCLUDES `index_nested` occurrences, so the index-nested `from` never makes +/// the reducer "hold the live reference": the whole reducer is frozen +/// (`PREVIOUS(sum(w[from]))`) and only the bare `from` stays live. (The retired +/// `expr0_contains_live_match` reproduced this by only inspecting subscript +/// HEADS; the occurrence switch reproduces it via `occ.index_nested`.) /// -/// This is the exact silent-drift the stage-2 occurrence switch must reproduce -/// via `occ.index_nested`. The scalar-element `SUM(w[from])` form (as opposed +/// The scalar-element `SUM(w[from])` form (as opposed /// to the array-slice `SUM(w[from, *])` production golden) is the one that /// compiles under BOTH the correct and the mutated behavior -- so nothing but /// this exact-text pin catches a selector that mishandles `index_nested` here: @@ -3121,6 +3122,7 @@ fn generate_link_score_equation_for_link_empty_target_is_err() { &all_vars, None, None, + &[], ); assert!( result.is_err(), @@ -3154,6 +3156,7 @@ fn generate_link_score_equation_for_link_normal_target_is_ok() { &all_vars, None, None, + &[], ) .expect("a normal scalar target must produce a valid link-score equation"); let text = match &equation { @@ -3311,6 +3314,7 @@ fn link_score_for_with_lookup_scalar_target_wraps_partial_in_lookup() { &all_vars, None, None, + &[], ) .expect("a with-lookup scalar target must produce a valid link-score equation"); let text = match &equation { @@ -3352,6 +3356,7 @@ fn link_score_for_with_lookup_a2a_target_pins_shared_table() { &all_vars, None, None, + &[], ) .expect("a with-lookup A2A target must produce a valid link-score equation"); let text = match &equation { @@ -3399,6 +3404,7 @@ fn link_score_for_with_lookup_arrayed_target_wraps_per_slot() { &all_vars, None, None, + &[], ) .expect("a with-lookup arrayed target must produce a valid link-score equation"); @@ -3452,6 +3458,7 @@ fn test_arrayed_link_score_population_to_migration_pressure_fixed_nyc() { &to_var, None, None, + &test_occurrences_for_var(&to_var, &from, &source_dim_elements), ) .unwrap(); @@ -3530,6 +3537,7 @@ fn test_arrayed_link_score_population_to_migration_pressure_fixed_boston() { &to_var, None, None, + &test_occurrences_for_var(&to_var, &from, &source_dim_elements), ) .unwrap(); @@ -3595,6 +3603,7 @@ fn test_arrayed_link_score_stock_to_flow_per_element_partials() { &births, None, None, + &test_occurrences_for_var(&births, &stock, &source_dim_elements), ) .unwrap(); @@ -3646,6 +3655,7 @@ fn test_scalar_and_a2a_link_scores_keep_their_shapes() { &scalar_to, None, None, + &[], ) .unwrap(); assert!( @@ -3692,6 +3702,7 @@ fn test_scalar_and_a2a_link_scores_keep_their_shapes() { &a2a_to, None, None, + &[], ) .unwrap(); match equation { @@ -4715,6 +4726,114 @@ fn test_body_aware_rank_keeps_delta_ratio() { ); } +/// Test wrapper around `shaped_guard_form_text`: reconstructs the occurrence IR +/// the production wrap consumes from the raw equation text (production callers +/// get it from `model_ltm_reference_sites`; slot-0 body) and threads it in, so +/// these focused text-in/text-out tests keep exercising the real chooser. +#[allow(clippy::too_many_arguments)] +fn sgft( + equation_text: &str, + deps: &HashSet>, + from: &Ident, + shape: &RefShape, + source_dim_elements: &[Vec], + source_dim_names: &[String], + iter_ctx: Option<&IteratedDimCtx<'_>>, + dims_ctx: Option<&crate::dimensions::DimensionsContext>, + target_ref: &str, + gf_table_ref: Option<&str>, +) -> Result { + let occ_sites = + build_wrap_test_occurrences(equation_text, from, deps, source_dim_elements, iter_ctx); + let occ = OccurrenceLookup::for_slot(&occ_sites, 0); + shaped_guard_form_text( + equation_text, + deps, + from, + shape, + source_dim_elements, + source_dim_names, + iter_ctx, + dims_ctx, + target_ref, + gf_table_ref, + &occ, + ) +} + +/// Finding 1 (loud-degradation, not silent-zero): a live-source subscript node +/// with NO occurrence at its tracked structural path -- on a NON-empty +/// occurrence stream -- is a walker desync (the reparsed-`Expr0` walk drifted +/// from the IR's `Expr2` `SiteId` numbering). On a miss the shape lookup returns +/// `None`, `node_shape == Some(live_shape)` fails, and the live reference would +/// be silently FROZEN -- a clean-compiling `PREVIOUS(...)` that zeroes the +/// score. That silent zero is exactly what the single-classifier flip must not +/// regress, so the miss surfaces LOUDLY (the `missing_occurrence` outcome and an +/// `Err` at the caller boundary, which the db-bearing emitters turn into a +/// skip-and-warn). +/// +/// `assert_occurrence_stream_aligns` proves the streams align corpus-wide, so a +/// miss is only reachable on a NOVEL production shape the gate cannot cover -- +/// which is precisely why the guard exists. It is simulated here by dropping the +/// live source's occurrence from an otherwise-aligned stream; its `helper` +/// co-dep survives, so the lookup stays NON-empty (the guard's `is_empty()` +/// scope, which excludes the legitimate source-free-slot miss, is satisfied). +#[test] +fn wrap_missing_live_source_occurrence_is_loud_not_silent_freeze() { + let equation = "pop[nyc] * helper"; + let deps = deps_set(&["helper"]); + let live = Ident::::new("pop"); + let shape = RefShape::FixedIndex(vec!["nyc".to_string()]); + let source_dims = vec![vec!["nyc".to_string(), "boston".to_string()]]; + + // A correctly-aligned stream, then drop pop's occurrence: the lookup stays + // non-empty (helper survives) but `pop[nyc]`'s node lookup now misses. + let desynced: Vec = + build_wrap_test_occurrences(equation, &live, &deps, &source_dims, None) + .into_iter() + .filter(|o| !matches!(&o.reference, OccurrenceRef::Variable(v) if v == "pop")) + .collect(); + let occ = OccurrenceLookup::for_slot(&desynced, 0); + assert!( + !occ.is_empty(), + "the guard scope requires a non-empty lookup -- helper must survive the drop" + ); + + let (_ast, out) = + wrap_changed_first_ast(equation, &deps, &live, &shape, None, None, None, &occ) + .expect("the equation parses"); + assert!( + out.missing_occurrence, + "a live-source subscript path-miss on a non-empty lookup must flag the desync" + ); + assert!( + out.live_ref.is_none(), + "the missed live source was frozen -- there is no live reference to normalize by" + ); + + // The flag becomes a LOUD `Err` at the shaped-guard caller boundary (the + // db-bearing emitters convert it to a skip-and-warn), NOT an Ok silent-zero + // changed-first partial. Changed-last is not even attempted: it would read + // the SAME desynced stream. + let err = shaped_guard_form_text( + equation, + &deps, + &live, + &shape, + &source_dims, + &["region".to_string()], + None, + None, + "combined", + None, + &occ, + ); + assert!( + err.is_err(), + "the missing-occurrence desync must surface as a loud Err, never Ok(silent-zero): {err:?}" + ); +} + // -- shaped_guard_form_text: attribution-convention chooser (GH #743) -- // // The chooser builds the standard changed-first guard form, but when the @@ -4743,7 +4862,7 @@ fn shaped_guard_form_falls_back_to_changed_last_for_unfreezable_co_source() { dim_ctx: None, dep_dims: None, }; - let text = shaped_guard_form_text( + let text = sgft( "SUM(matrix[D1, *] * frac[D1])", &deps, &live, @@ -4775,7 +4894,7 @@ fn shaped_guard_form_keeps_changed_first_when_freezable() { // `population / SUM(population[*])`: the live ref is OUTSIDE the // reducer occurrence that matters, and the whole reducer is frozen as // `PREVIOUS(sum(population[*]))` -- PREVIOUS of a scalar, freezable. - let text = shaped_guard_form_text( + let text = sgft( "population / SUM(population[*])", &deps, &live, @@ -4826,7 +4945,7 @@ fn shaped_guard_form_keeps_changed_first_when_freezable() { fn shaped_guard_form_rank_slice_arg_falls_back_to_changed_last() { let deps = deps_set(&["matrix", "frac"]); let live = Ident::::new("frac"); - let text = shaped_guard_form_text( + let text = sgft( "frac * RANK(matrix[d1, *], 1)", &deps, &live, @@ -4859,7 +4978,7 @@ fn shaped_guard_form_rank_slice_arg_falls_back_to_changed_last() { fn shaped_guard_form_errs_when_both_conventions_unfreezable() { let deps = deps_set(&["arr", "brr"]); let live = Ident::::new("arr"); - let err = shaped_guard_form_text( + let err = sgft( "SUM(arr[*] * brr[d1, *])", &deps, &live, @@ -4883,7 +5002,7 @@ fn shaped_guard_form_errs_when_both_conventions_unfreezable() { fn shaped_guard_form_errs_when_no_live_occurrence_to_freeze() { let deps = deps_set(&["matrix"]); let live = Ident::::new("frac"); - let err = shaped_guard_form_text( + let err = sgft( // A naked (non-reducer-enclosed) wildcard slice; `frac` absent. // Not a compilable model equation, but exercises the guard. "matrix[d1, *] * 2", @@ -5036,7 +5155,7 @@ fn shaped_guard_form_declines_bare_arrayed_feeder_of_unhoisted_reducer() { dim_ctx: None, dep_dims: None, }; - let err = shaped_guard_form_text( + let err = sgft( "SUM(matrix[D1, *] * frac)", &deps, &live, @@ -5072,7 +5191,7 @@ fn shaped_guard_form_scalar_feeder_inside_reducer_not_declined_by_gh779() { let deps = deps_set(&["matrix", "scale"]); let live = Ident::::new("scale"); // `scale` SCALAR -> empty source dims, so the GH #779 gate is inert. - let result = shaped_guard_form_text( + let result = sgft( "SUM(matrix[D1, *] * scale)", &deps, &live, diff --git a/src/simlin-engine/src/ltm_augment_wrap_test_support.rs b/src/simlin-engine/src/ltm_augment_wrap_test_support.rs new file mode 100644 index 000000000..01884df6c --- /dev/null +++ b/src/simlin-engine/src/ltm_augment_wrap_test_support.rs @@ -0,0 +1,273 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! `#[cfg(test)]` occurrence-IR reconstruction for the ceteris-paribus wrap +//! unit tests. +//! +//! Production drives the wrap from the salsa occurrence IR +//! (`db::ltm_ir::model_ltm_reference_sites`). The focused text-in/text-out wrap +//! unit tests (`build_partial_equation_shaped` and the direct generator tests) +//! have no db, so these helpers rebuild an equivalent per-slot occurrence stream +//! on the reparsed `Expr0` -- using the `#[cfg(test)]` Expr0 classifiers the +//! alignment gate (`ltm_classifier_agreement_tests`) proves stay in step with +//! the IR -- so those tests keep exercising the real production wrap with +//! byte-identical inputs. Split out of `ltm_augment.rs` only to keep that file +//! under the project line-count lint; included via `#[path]`, so `super::*` +//! resolves the parent's private items. + +use super::*; + +/// Reconstruct the occurrence-IR stream ([`OccurrenceSite`]s) for one target +/// equation from its raw text, for the `#[cfg(test)]` wrap unit tests +/// ([`build_partial_equation_shaped`]). Production gets the stream from +/// `db::ltm_ir::model_ltm_reference_sites` (the `Expr2` walk); this rebuilds an +/// equivalent stream on the reparsed `Expr0` so those focused text-in/text-out +/// tests keep exercising the real production wrap with byte-identical inputs. +/// +/// The paths start at slot `0` (mirroring `walk_all_in_expr`'s single-body slot +/// push), so [`OccurrenceLookup::for_slot`]`(.., 0)` rebases them to the +/// slot-local paths the wrap tracks. Occurrences are recorded for every ident in +/// `deps` ∪ `{live_source}` (the idents the wrap acts on); the live source's +/// shape/axes come from the Expr0 classifier, an other-dep's axes from +/// [`other_dep_occurrence_axes`] (only its verdict is consumed). +#[cfg(test)] +pub(crate) fn build_wrap_test_occurrences( + equation_text: &str, + live_source: &Ident, + deps: &HashSet>, + source_dim_elements: &[Vec], + iter_ctx: Option<&IteratedDimCtx<'_>>, +) -> Vec { + let Ok(Some(ast)) = Expr0::new(equation_text, LexerType::Equation) else { + return Vec::new(); + }; + let mut recorded = deps.clone(); + recorded.insert(live_source.clone()); + let mut out = Vec::new(); + // Slot 0 for a scalar/A2A body (matching `walk_all_in_expr`). + let mut path = vec![0u16]; + walk_wrap_test_occurrences( + &ast, + live_source, + &recorded, + source_dim_elements, + iter_ctx, + false, + &mut path, + &mut out, + ); + out +} + +/// Build the occurrence stream for a whole target `Variable` (all slots), for +/// the `#[cfg(test)]` generator tests that construct a `Variable` directly (no +/// db). Mirrors `walk_all_in_expr`'s slot numbering: a scalar/A2A body is slot +/// `0`; an `Ast::Arrayed` target's per-element slots are canonical +/// element-key-sorted, then the default. Records references to `from` (the live +/// source); those tests reference only the source, and `iter_ctx = None` is +/// sufficient for their FixedIndex / Bare shapes. +#[cfg(test)] +pub(crate) fn test_occurrences_for_var( + to_var: &Variable, + from: &Ident, + source_dim_elements: &[Vec], +) -> Vec { + use crate::ast::Ast; + let recorded: HashSet> = std::iter::once(from.clone()).collect(); + let mut out = Vec::new(); + let Some(ast) = to_var.ast() else { + return out; + }; + let walk_slot = |text: &str, slot: u16, out: &mut Vec| { + if let Ok(Some(e0)) = Expr0::new(text, LexerType::Equation) { + let mut path = vec![slot]; + walk_wrap_test_occurrences( + &e0, + from, + &recorded, + source_dim_elements, + None, + false, + &mut path, + out, + ); + } + }; + match ast { + Ast::Scalar(expr) | Ast::ApplyToAll(_, expr) => { + walk_slot(&crate::patch::expr2_to_string(expr), 0, &mut out); + } + Ast::Arrayed(_, per_elem, default_expr, _) => { + let mut keys: Vec<_> = per_elem.keys().collect(); + keys.sort(); + for (slot, k) in keys.iter().enumerate() { + walk_slot( + &crate::patch::expr2_to_string(&per_elem[*k]), + slot as u16, + &mut out, + ); + } + if let Some(def) = default_expr { + walk_slot( + &crate::patch::expr2_to_string(def), + keys.len() as u16, + &mut out, + ); + } + } + } + out +} + +/// One synthesized test occurrence: only `site_id`, `reference`, `shape`, +/// `axes`, and `index_nested` are read by the wrap; the rest carry inert +/// defaults. +#[cfg(test)] +fn test_occurrence( + ident: &Ident, + path: &[u16], + shape: RefShape, + axes: Vec, + index_nested: bool, +) -> OccurrenceSite { + OccurrenceSite { + site_id: crate::db::ltm_ir::SiteId(path.to_vec().into_boxed_slice()), + reference: OccurrenceRef::Variable(ident.as_str().to_string()), + shape, + axes, + target_element: None, + routing: crate::db::ltm_ir::OccurrenceRouting::Direct, + in_reducer: false, + reducer_keys: Vec::new(), + already_lagged: false, + index_nested, + } +} + +/// Recursive walker for [`build_wrap_test_occurrences`], mirroring +/// `db::ltm_ir::walk_all_in_expr`'s child-index path construction (and its +/// sticky `index_nested` flag) so the synthesized occurrences land at the +/// SiteId paths the wrap tracks and carry the `index_nested` bit +/// [`OccurrenceLookup::subtree_has_live_shape`] filters on. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +fn walk_wrap_test_occurrences( + expr: &Expr0, + live_source: &Ident, + recorded: &HashSet>, + source_dim_elements: &[Vec], + iter_ctx: Option<&IteratedDimCtx<'_>>, + index_nested: bool, + path: &mut Vec, + out: &mut Vec, +) { + let recurse = |sub: &Expr0, index_nested: bool, path: &mut Vec, out: &mut _| { + walk_wrap_test_occurrences( + sub, + live_source, + recorded, + source_dim_elements, + iter_ctx, + index_nested, + path, + out, + ); + }; + match expr { + Expr0::Const(..) => {} + Expr0::Var(name, _) => { + let c = Ident::::new(name.as_str()); + if recorded.contains(&c) { + out.push(test_occurrence( + &c, + path, + RefShape::Bare, + Vec::new(), + index_nested, + )); + } + } + Expr0::Subscript(name, indices, _) => { + let c = Ident::::new(name.as_str()); + if recorded.contains(&c) { + let (shape, axes) = if &c == live_source { + let shape = + classify_expr0_subscript_shape(indices, source_dim_elements, iter_ctx); + let axes = indices + .iter() + .enumerate() + .map(|(i, idx)| { + match resolve_literal_element_index(idx, i, source_dim_elements) { + Some(e) => OccurrenceAxis::Pinned(e), + None => OccurrenceAxis::Dynamic, + } + }) + .collect(); + (shape, axes) + } else { + // Other dep: only the verdict (from its axes) is consumed. + ( + RefShape::Bare, + other_dep_occurrence_axes(&c, indices, iter_ctx), + ) + }; + out.push(test_occurrence(&c, path, shape, axes, index_nested)); + } + // Descending into a subscript index sets `index_nested` sticky-true. + for (i, idx) in indices.iter().enumerate() { + path.push(i as u16); + match idx { + IndexExpr0::Expr(e) => recurse(e, true, path, out), + IndexExpr0::Range(l, r, _) => { + path.push(0); + recurse(l, true, path, out); + path.pop(); + path.push(1); + recurse(r, true, path, out); + path.pop(); + } + IndexExpr0::Wildcard(_) + | IndexExpr0::StarRange(_, _) + | IndexExpr0::DimPosition(_, _) => {} + } + path.pop(); + } + } + Expr0::App(UntypedBuiltinFn(fname, args), _) => { + let lname = fname.to_ascii_lowercase(); + let skip_first = matches!( + lname.as_str(), + "lookup" | "lookup_forward" | "lookup_backward" + ) && !args.is_empty(); + for (i, a) in args.iter().enumerate() { + if skip_first && i == 0 { + continue; + } + path.push(i as u16); + recurse(a, index_nested, path, out); + path.pop(); + } + } + Expr0::Op1(_, inner, _) => { + path.push(0); + recurse(inner, index_nested, path, out); + path.pop(); + } + Expr0::Op2(_, l, r, _) => { + path.push(0); + recurse(l, index_nested, path, out); + path.pop(); + path.push(1); + recurse(r, index_nested, path, out); + path.pop(); + } + Expr0::If(c, t, e, _) => { + for (child, sub) in [c, t, e].into_iter().enumerate() { + path.push(child as u16); + recurse(sub, index_nested, path, out); + path.pop(); + } + } + } +} diff --git a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs index c4cc22e7a..2304cff06 100644 --- a/src/simlin-engine/src/ltm_classifier_agreement_tests.rs +++ b/src/simlin-engine/src/ltm_classifier_agreement_tests.rs @@ -2,34 +2,34 @@ // Use of this source code is governed by the Apache License, // Version 2.0, that can be found in the LICENSE file. -//! Track-A differential gate: the two LTM access-shape classifier families -//! agree on every non-reducer reference occurrence. +//! Track-A differential gate: the `#[cfg(test)]` Expr0 access-shape classifier +//! stays in step with the production occurrence IR on every non-reducer +//! reference occurrence. //! -//! # The failure mode this gate retires +//! # What this gate now guards //! -//! An LTM causal edge's access shape is decided TWICE, on two ASTs, by two -//! mirrored classifier families: +//! PRODUCTION decides an LTM causal edge's access shape ONCE, on the target's +//! `Expr2` AST ([`crate::db::ltm_ir`]: `resolve_literal_index` / +//! `classify_subscript_shape` / `classify_iterated_dim_shape`, composed by the +//! walker `collect_all_reference_sites` and cached in the salsa query +//! `model_ltm_reference_sites`). Both consumers -- causal-edge emission AND the +//! ceteris-paribus wrap -- read that single classification: the wrap tracks each +//! occurrence's structural `SiteId` path and looks its shape up in +//! `model_ltm_reference_sites`' occurrence stream (`OccurrenceLookup`), so it can +//! no longer drift from the edge emitter (the historical two-family silent-zero +//! class -- GH #759 dimension-name indices, GH #913 printer/parser asymmetry, the +//! `pop[01]` indexed-literal canonicalization -- is structurally impossible). //! -//! * **Expr2 side** ([`crate::db::ltm_ir`]) -- `resolve_literal_index` / -//! `classify_subscript_shape` / `classify_iterated_dim_shape`, composed by -//! the production walker `collect_all_reference_sites` and consumed through -//! the salsa query `model_ltm_reference_sites`. This drives causal-edge -//! emission and the per-shape link-score selection: `emit_per_shape_link_scores` -//! feeds each site's `shape` into the partial builder as `live_shape`. -//! * **Expr0 side** ([`super`]) -- `resolve_literal_element_index` / -//! `classify_expr0_subscript_shape` / `classify_expr0_per_element_axes` / -//! `is_live_source_iterated_dim_subscript`, consumed by -//! `wrap_non_matching_in_previous` on the *printed-and-reparsed* target -//! equation text. This re-derives each occurrence's shape and keeps live -//! only the occurrences whose shape EQUALS `live_shape`. -//! -//! If the families drift, no Expr0 occurrence matches `live_shape`, every -//! reference to the source is PREVIOUS-wrapped, and the link score silently -//! collapses to a constant (see the "must stay in sync" docstring on -//! `super::resolve_literal_element_index`). Historical drift bugs, all of -//! this class: GH #759 (dimension-name indices), GH #913 (printer/parser -//! asymmetry dropping attribution), the `pop[01]` indexed-literal -//! canonicalization. +//! The Expr0 classifier family (`resolve_literal_element_index` / +//! `classify_expr0_subscript_shape` / `classify_expr0_per_element_axes` / +//! `is_live_source_iterated_dim_subscript` in [`super`]) now survives only +//! `#[cfg(test)]`: it reconstructs an occurrence stream on the printed-and-reparsed +//! target text for the focused wrap unit tests (`ltm_augment_wrap_test_support`). +//! Those tests exercise the real production wrap, so the reconstructed stream +//! must carry the same shapes/paths the IR would. This gate is what proves it: +//! it classifies each occurrence with BOTH families over the corpus and asserts +//! they agree, so the `#[cfg(test)]` reconstruction cannot silently diverge from +//! the IR and feed the wrap a shape production never would. //! //! # What this gate compares, and why it is drift-complete //! @@ -72,12 +72,13 @@ //! their shape is inconsequential. A `Direct`, not-hoisted reducer argument //! (the dynamic-index carve-out, `SUM(pop[idx,*])`) keeps its shape all the //! way to `emit_per_shape_link_scores`, which feeds it as `live_shape` into -//! the partial builder; `wrap_non_matching_in_previous`'s reducer arm then -//! keeps the reducer live only if the in-reducer occurrence's Expr0 shape -//! (`expr0_contains_live_match` -> `classify_expr0_subscript_shape`) EQUALS -//! that `live_shape`, so a drift on that occurrence WOULD silently zero the -//! score. The exclusion is sound for a different reason: both classifier -//! families are reducer-context-FREE (`classify_subscript_shape` / +//! the wrap; the wrap's reducer arm then keeps the reducer live only if the +//! in-reducer occurrence's IR shape EQUALS that `live_shape` (production reads +//! it from the occurrence lookup; the `#[cfg(test)]` reconstruction classifies +//! it with `classify_expr0_subscript_shape`), so a reconstruction drift on +//! that occurrence would feed the wrap a shape the IR never produced. The +//! exclusion is sound for a different reason: both classifier families are +//! reducer-context-FREE (`classify_subscript_shape` / //! `classify_iterated_dim_shape` / `resolve_literal_index` on Expr2, //! `classify_expr0_subscript_shape` / `resolve_literal_element_index` on //! Expr0 -- none takes an `in_reducer` input), so any drift on a @@ -146,7 +147,8 @@ use crate::db::ltm_ir::{ model_ltm_reference_sites, }; use crate::db::{ - SimlinDb, project_dimensions_context, reconstruct_model_variables, sync_from_datamodel, + DiagnosticSeverity, SimlinDb, collect_all_diagnostics, project_dimensions_context, + reconstruct_model_variables, sync_from_datamodel, }; use crate::dimensions::{Dimension, DimensionsContext}; use crate::ltm_agg::{AxisRead, ReducerKind, reducer_kind_from_name}; @@ -165,6 +167,12 @@ struct Expr0Occurrence { source: String, indices: Option>, in_reducer: bool, + /// The structural child-index path from the target's slot root down to + /// this occurrence node, built to mirror `walk_all_in_expr`'s `SiteId` + /// construction exactly (slot prefix + descent chain). Stage 2's wrap + /// consults the occurrence IR by this path, so it must equal the IR's + /// `SiteId`; the alignment sweep pins that. + path: Vec, } /// Whether `name`/`arity` names a reducer whose references route through an @@ -208,6 +216,7 @@ fn collect_expr0_occurrences( expr: &Expr0, variables: &HashMap, Variable>, in_reducer: bool, + path: &mut Vec, out: &mut Vec, ) { match expr { @@ -219,6 +228,7 @@ fn collect_expr0_occurrences( source: canonical.as_str().to_string(), indices: None, in_reducer, + path: path.clone(), }); } } @@ -229,6 +239,7 @@ fn collect_expr0_occurrences( source: canonical.as_str().to_string(), indices: Some(indices.clone()), in_reducer, + path: path.clone(), }); } // Mirror the Expr2 walker's element-selector skip: an index that @@ -252,16 +263,24 @@ fn collect_expr0_occurrences( if resolve_literal_element_index(idx, i, &source_dim_elements).is_some() { continue; } + path.push(i as u16); match idx { - IndexExpr0::Expr(e) => collect_expr0_occurrences(e, variables, in_reducer, out), + IndexExpr0::Expr(e) => { + collect_expr0_occurrences(e, variables, in_reducer, path, out) + } IndexExpr0::Range(l, r, _) => { - collect_expr0_occurrences(l, variables, in_reducer, out); - collect_expr0_occurrences(r, variables, in_reducer, out); + path.push(0); + collect_expr0_occurrences(l, variables, in_reducer, path, out); + path.pop(); + path.push(1); + collect_expr0_occurrences(r, variables, in_reducer, path, out); + path.pop(); } IndexExpr0::Wildcard(_) | IndexExpr0::StarRange(_, _) | IndexExpr0::DimPosition(_, _) => {} } + path.pop(); } } Expr0::App(UntypedBuiltinFn(name, args), _) => { @@ -275,18 +294,30 @@ fn collect_expr0_occurrences( if skip_first && i == 0 { continue; } - collect_expr0_occurrences(arg, variables, child_in_reducer, out); + path.push(i as u16); + collect_expr0_occurrences(arg, variables, child_in_reducer, path, out); + path.pop(); } } - Expr0::Op1(_, inner, _) => collect_expr0_occurrences(inner, variables, in_reducer, out), + Expr0::Op1(_, inner, _) => { + path.push(0); + collect_expr0_occurrences(inner, variables, in_reducer, path, out); + path.pop(); + } Expr0::Op2(_, l, r, _) => { - collect_expr0_occurrences(l, variables, in_reducer, out); - collect_expr0_occurrences(r, variables, in_reducer, out); + path.push(0); + collect_expr0_occurrences(l, variables, in_reducer, path, out); + path.pop(); + path.push(1); + collect_expr0_occurrences(r, variables, in_reducer, path, out); + path.pop(); } Expr0::If(cond, then_e, else_e, _) => { - collect_expr0_occurrences(cond, variables, in_reducer, out); - collect_expr0_occurrences(then_e, variables, in_reducer, out); - collect_expr0_occurrences(else_e, variables, in_reducer, out); + for (child, sub) in [cond, then_e, else_e].into_iter().enumerate() { + path.push(child as u16); + collect_expr0_occurrences(sub, variables, in_reducer, path, out); + path.pop(); + } } } } @@ -314,12 +345,13 @@ fn target_iterated_dims_of(to_var: &Variable) -> Vec { } } -/// Classify one reparsed Expr0 occurrence of `source` with the production -/// Expr0 classifier, using the SAME context construction the production caller -/// builds. A bare occurrence is `Bare` by construction (matching the Expr2 -/// walker's `Var` arm). `dep_dims` is `None`: it feeds only the *non-live-dep* -/// collapse in `classify_other_dep_iterated_dim_subscript`, never the live -/// source's shape, which is all this classifies. +/// Classify one reparsed Expr0 occurrence of `source` with the `#[cfg(test)]` +/// Expr0 classifier, using the SAME context construction the production wrap's +/// occurrence lookup was built against. A bare occurrence is `Bare` by +/// construction (matching the Expr2 walker's `Var` arm). `dep_dims` is `None`: +/// it feeds only the *non-live-dep* verdict (`other_dep_occurrence_axes` + +/// `derive_other_dep_verdict`), never the live source's shape, which is all +/// this classifies. fn classify_expr0_occurrence( occ: &Expr0Occurrence, to_var: &Variable, @@ -364,17 +396,21 @@ fn expr0_occurrences_for_target( let Some(ast) = to_var.ast() else { return out; }; - let mut walk_text = |text: &str| { + // `slot` is the first `SiteId` path element (`walk_all_in_expr`: slot 0 for + // a scalar/A2A body, the sorted element index for each `Ast::Arrayed` slot, + // then the default slot after the last element). + let mut walk_text = |slot: u16, text: &str| { let parsed = Expr0::new(text, crate::lexer::LexerType::Equation).unwrap_or_else(|e| { panic!("target '{to_name}' printed text failed to reparse (GH #913 drift class): text={text:?}, err={e:?}") }); if let Some(expr0) = parsed { - collect_expr0_occurrences(&expr0, variables, false, &mut out); + let mut path = vec![slot]; + collect_expr0_occurrences(&expr0, variables, false, &mut path, &mut out); } }; match ast { Ast::Scalar(expr) | Ast::ApplyToAll(_, expr) => { - walk_text(&crate::patch::expr2_to_string(expr)); + walk_text(0, &crate::patch::expr2_to_string(expr)); } Ast::Arrayed(_, per_elem, default_expr, _) => { // Deterministic slot order (matches the sorted walk in @@ -382,11 +418,11 @@ fn expr0_occurrences_for_target( // per-edge multiset but keeps failure output stable. let mut keys: Vec<_> = per_elem.keys().collect(); keys.sort(); - for k in keys { - walk_text(&crate::patch::expr2_to_string(&per_elem[k])); + for (slot, k) in keys.iter().enumerate() { + walk_text(slot as u16, &crate::patch::expr2_to_string(&per_elem[*k])); } if let Some(default) = default_expr { - walk_text(&crate::patch::expr2_to_string(default)); + walk_text(keys.len() as u16, &crate::patch::expr2_to_string(default)); } } } @@ -502,6 +538,19 @@ fn assert_occurrence_stream_aligns( ir_src, &e0_occ.source, "occurrence-stream SOURCE mismatch for target '{to_str}' (IR vs reparsed Expr0)" ); + // The SiteId-keyed bridge stage 2 consumes: the wrap looks the + // occurrence up by the structural child-index path it walks. That path + // must equal the IR's `SiteId`, computed on the Expr2 AST -- so the + // print->reparse round trip must be child-index-isomorphic at every + // occurrence node. A mismatch would make the wrap's SiteId lookup miss + // (or hit the wrong occurrence) despite the streams aligning positionally. + assert_eq!( + &ir_occ.site_id.0[..], + &e0_occ.path[..], + "occurrence-stream SiteId PATH mismatch for {ir_src} -> {to_str}: the reparsed-Expr0 \ + child-index path does not equal the IR's Expr2 SiteId, so a SiteId-keyed wrap lookup \ + would desync" + ); assert_eq!( ir_occ.in_reducer, e0_occ.in_reducer, "occurrence-stream in_reducer mismatch for {ir_src} -> {to_str}" @@ -530,15 +579,33 @@ fn assert_occurrence_streams_align(tp: &TestProject) { let model = sync.models["main"].source; let project = sync.project; + // Non-vacuity guard 1 (fail loudly, never skip silently): a fixture + // equation that fails to parse leaves its target with NO AST, so + // `expr0_occurrences_for_target` returns empty and every alignment + // assertion below passes trivially (0 == 0). Catch that at the source: an + // Error-severity diagnostic surfaces exactly a parse failure (the + // grammar-rejected `IF`-as-operand shape was the concrete instance). + let errors: Vec<_> = collect_all_diagnostics(&db, project) + .into_iter() + .filter(|d| d.severity == DiagnosticSeverity::Error) + .collect(); + assert!( + errors.is_empty(), + "alignment fixture has compile errors -- a target that fails to parse would \ + silently vacate the alignment sweep (empty occurrence stream): {errors:?}" + ); + let variables = reconstruct_model_variables(&db, model, project); let dim_ctx = project_dimensions_context(&db, project); let ir = model_ltm_reference_sites(&db, model, project); + let mut total_occurrences = 0usize; let mut to_names: Vec<&Ident> = variables.keys().collect(); to_names.sort(); for to_name in to_names { let to_var = &variables[to_name]; let expr0_occs = expr0_occurrences_for_target(to_name.as_str(), to_var, &variables); + total_occurrences += expr0_occs.len(); assert_occurrence_stream_aligns( to_name.as_str(), to_var, @@ -548,6 +615,18 @@ fn assert_occurrence_streams_align(tp: &TestProject) { &expr0_occs, ); } + + // Non-vacuity guard 2: even with every equation parsing, a fixture whose + // targets reference no model variables (all-constant) produces an empty + // occurrence stream model-wide, so the per-target length assertions all + // compare 0 == 0. Every alignment fixture exists to exercise the SiteId-path + // / shape zip on REAL occurrences, so require at least one. + assert!( + total_occurrences > 0, + "alignment fixture produced NO occurrences model-wide: the SiteId-path / shape \ + assertions never run (every per-target stream is empty, so the length checks \ + pass 0 == 0)" + ); } #[test] @@ -569,6 +648,39 @@ fn align_already_lagged_and_index_nested_streams() { assert_occurrence_streams_align(&tp); } +#[test] +fn align_precedence_and_nesting_streams() { + // The SiteId-zip bridge assumes the print->reparse round trip is + // child-index-isomorphic to the Expr2 AST. Operator precedence, + // parenthesization, unary minus, and `IF THEN ELSE` nesting are where a + // printer/parser asymmetry would reshape the tree and desync the SiteId + // path even when the flat occurrence stream still aligns positionally, so + // stress those explicitly (the `align_*` fixtures assert full SiteId-path + // equality, not just positional order). + let tp = TestProject::new("main") + .with_sim_time(0.0, 3.0, 1.0) + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "100") + .scalar_aux("a", "1") + .scalar_aux("b", "2") + .scalar_aux("c", "3") + .scalar_aux("d", "4") + .array_aux( + "mixed[Region]", + // The `IF ... THEN ... ELSE ...` operand is parenthesized because + // the grammar accepts `IF` only at the head of an expression, never + // as a bare binary operand (`parser::parse_expr`): the unparenthesized + // form fails to parse, leaving `mixed` with no AST and an EMPTY + // occurrence stream -- which would vacate this whole fixture (the + // alignment loop would compare 0 == 0 and never run the SiteId-path / + // shape assertions the fixture exists to stress). The parenthesized + // form round-trips through the printer (an `If` child under an `Op2` + // parent is always re-parenthesized, `ast::paren_if_necessary`). + "a * (b + c) - -d + pop / (a - b * c) + (IF a > b THEN pop * c ELSE d - a)", + ); + assert_occurrence_streams_align(&tp); +} + #[test] fn align_reducer_and_lookup_streams() { // A `LOOKUP` table argument is skipped by both walkers; reducer arguments From dc20563e8555ca1836c5c6861225776491bfd00b Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 07:56:50 -0700 Subject: [PATCH 06/14] scripts/codex-review: 5.6-sol --- scripts/codex-review | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/codex-review b/scripts/codex-review index be9205e69..1d5344add 100755 --- a/scripts/codex-review +++ b/scripts/codex-review @@ -2,7 +2,7 @@ set -euo pipefail codex \ - -c 'model="gpt-5.5"' \ + -c 'model="gpt-5.6-sol"' \ -c 'model_reasoning_effort="xhigh"' \ exec review --json --base origin/main \ | tee /tmp/codex.stdout \ From dea42188e44c5e7e3f201a846c4cd9db3728d58c Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 08:03:32 -0700 Subject: [PATCH 07/14] engine: store LTM synthetic equations as typed ASTs LTM synthetic variables were stored as datamodel::Equation text: the transform built a wrapped Expr0 AST, printed it, and db/ltm/parse.rs minted a transient datamodel aux and re-parsed the text for compilation -- 2-3 parses per equation and a printer/parser asymmetry surface at the compile boundary. LtmSyntheticVar now carries the typed equation (a new LtmEquation in db/ltm/equation.rs holding the wrapped Expr0 arms for scalar, apply-to-all, and arrayed forms); producers hand over the AST they already built, and compilation consumes it directly. Printing survives only as a diagnostics projection (warnings, debug dumps, the characterization goldens), which is why the goldens are byte-identical. Two long-lived seams die at the same boundary. The legacy link_score_equation_text query is deleted: compile.rs's sub-case (a) previously compiled fragments from the legacy text while emitters reported the shaped query's text (with a known dim_ctx divergence between them); compiled and reported equations now come from one stored derivation by construction. module_link_score_equation's identifier_set fallback (and its loud marker) is deleted: the occurrence IR is the only source for module-output live selection, and a missing occurrence takes the loud missing-occurrence path. Salsa incrementality is preserved: the shaped text query backdates on the PartialEq-equal typed equation, so unrelated edits reuse the expensive fragment (pinned by the existing per-link caching test); the units-context dependency was dropped from the LTM parse path entirely. Mutation probes confirm the compiled representation is test-bound: a perturbed stored AST trips numeric pins even though text goldens are blind to it. This is the storage/compile half of the GH #965 typed-equation work; the generation half (the transform still parses the target equation from printed text once at the source boundary) is the follow-on. --- src/simlin-engine/CLAUDE.md | 9 +- .../examples/ltm_helper_census.rs | 22 +- src/simlin-engine/src/db.rs | 250 +++------------ src/simlin-engine/src/db/ltm/compile.rs | 97 ++++-- src/simlin-engine/src/db/ltm/equation.rs | 292 ++++++++++++++++++ src/simlin-engine/src/db/ltm/link_scores.rs | 47 +-- src/simlin-engine/src/db/ltm/loops.rs | 4 +- src/simlin-engine/src/db/ltm/mod.rs | 26 +- src/simlin-engine/src/db/ltm/parse.rs | 216 ++++++------- src/simlin-engine/src/db/ltm_char_tests.rs | 159 ++++++++-- src/simlin-engine/src/db/ltm_module_tests.rs | 14 +- src/simlin-engine/src/db/ltm_tests.rs | 128 ++++---- src/simlin-engine/src/db/ltm_unified_tests.rs | 4 +- src/simlin-engine/src/db/tests.rs | 5 +- src/simlin-engine/src/ltm_agg.rs | 2 +- src/simlin-engine/src/ltm_augment.rs | 80 +++-- src/simlin-engine/src/ltm_augment_tests.rs | 85 ++--- src/simlin-engine/src/ltm_finding_tests.rs | 16 +- src/simlin-engine/src/ltm_post.rs | 24 +- src/simlin-engine/src/variable.rs | 2 +- .../tests/integration/ltm_array_agg.rs | 44 +-- .../tests/integration/simulate_ltm.rs | 11 +- 22 files changed, 886 insertions(+), 651 deletions(-) create mode 100644 src/simlin-engine/src/db/ltm/equation.rs diff --git a/src/simlin-engine/CLAUDE.md b/src/simlin-engine/CLAUDE.md index 737ff29b6..4a244b6a2 100644 --- a/src/simlin-engine/CLAUDE.md +++ b/src/simlin-engine/CLAUDE.md @@ -101,7 +101,7 @@ Diagnostics (`collect_all_diagnostics`) run on the user's `SourceProject`, so sy The primary compilation path uses salsa tracked functions for fine-grained incrementality. Key modules: -- **`src/db.rs`** - The module ROOT of the salsa pipeline. After the db-submodule split it holds only the database itself (`SimlinDb` + its `Db` trait, the sync/staged/restore/`current_source_project` methods, the `StdlibModels` cache, the `impl salsa::Database`/`Db`), the `compile_project_incremental()` production entry point, the LTM-glue surface (`LtmSyntheticVar`/`LtmVariablesResult` result types + the `link_score_equation_text` salsa query and its `module_link_score_equation` (the shared module-link helper -- composite reference / ceteris-paribus / signed `black_box_unit_transfer_equation` fallback, also called by `link_score_equation_text_shaped` so the two twins never drift; `module_composite_ports` reads the sub-model's `model_ltm_variables` to decide whether a port has a composite, so the same composite reference is used in exhaustive AND discovery mode)/`find_model_output_ports_for_module`/`module_input_pathways_from_edges`/`generate_max_abs_selection` helpers; the last folds a module input port's "strongest pathway" composite through O(1)-sized accumulator helper variables -- inlining the fold into one nested expression doubles the equation text per pathway, which OOMed on macro modules with hundreds of pathways. Since PR #684 a stockless **passthrough** with input→output pathways ALSO emits pathway/composite vars, and since GH #748 the stock-free early return in `model_ltm_variables` fires only when the model is genuinely STATELESS -- no parent-level stocks, no input ports, and no (transitively) stock-carrying module instance (`modules_carry_state`), so a module-only root whose state lives inside a SMOOTH/DELAY or user sub-model is scored instead of silently emitting nothing, and the exhaustive-mode loop-score builder overrides each `x→m` module link with a **per-exit-port pathway selection** (`compute_module_link_overrides` in `src/db/ltm/mod.rs` emits a `$⁚ltm⁚link_score⁚{x}→{m}⁚via⁚{exit}` alias selecting the pathway the loop actually traverses, threaded into `generate_loop_score_variables` as a `(loop_id, link_index)` side-table) because the composite max-abs-selects across ALL pathways and so picks the wrong output port for a multi-output module; `find_model_output_ports` now sorts its result (GH #680) so the parent's pathway recomputation matches the sub-model's emitted `$⁚ltm⁚path⁚{port}⁚{idx}` indices, the residual being that pinned loops pass an empty override map), the `set_project_ltm_enabled`/`set_project_ltm_discovery_mode` setters plus the `LtmEnabledGuard` RAII scope guard (transiently flips `ltm_enabled` and unconditionally restores it on drop; shared by libsimlin's `simlin_project_get_errors` / from-wasm FFIs (GH #466) and `simlin-mcp-core`'s `read_model`/`edit_model` diagnostic passes (GH #662) so the LTM-only advisory harvest behaves identically across every diagnostic-collection surface), and the `#[cfg(test)]` test-module mounts. Everything else moved into per-concern db submodules (`input`/`query`/`sync`/`diagnostic`/`layout`/`var_fragment`/`fragment_compile`/`assemble`/`dep_graph`/`analysis`/`ltm`/...) and is re-exported at the root so the historical `simlin_engine::db::...` surface (and the `use super::*` globs in the root-mounted test modules and `implicit_deps.rs`) keep resolving. `SimlinDb` OWNS its sync state (a private `sync_state: Option` field; the `#[salsa::db]` macro finds `storage` by type, and the field is only mutated via `&mut self` during sync, never during parallel query execution, so no interior mutability is needed), so incrementality is automatic: `db.sync(project) -> SourceProject` threads the prior handles internally (a no-op re-sync stays a salsa cache hit), `db.sync_staged(project) -> (SourceProject, Option)` additionally returns the PRE-staging handles for an exact rollback, `db.restore(project, prev)` re-syncs the prior datamodel with those handles, and `db.current_source_project()` reads the latest handle. A SECOND, `pub(crate)` slot -- `expanded_state: Option`, driven by `db.sync_expanded(expanded_datamodel)` -- holds the conveyor/queue-expanded twin of the project so the special-stock build path is incremental too (see "Conveyors and queues"); it stays `None` until a special-stock model is first built, is never cleared thereafter (salsa cannot reclaim inputs, so clearing would only force a second set to be minted), and its handle is reachable only as `sync_expanded`'s return value, which is what keeps a rolled-back staged patch from poisoning it. `sync_from_datamodel` (fresh `&db` path returning `SyncResult`) and `sync_from_datamodel_incremental` stay public as the internal mechanism the methods wrap and for the read-only test path; `collect_all_diagnostics(db, project: SourceProject)` iterates `project.models(db)`. `SourceProject` carries `ltm_enabled` and `ltm_discovery_mode` flags for LTM compilation, plus `macro_declarations: Vec<(String, Option)>` -- the ordered, pre-dedup list of project models' canonical names + macro specs that the demand-driven `project_macro_registry` query reads to re-derive the macro build error (the canonical-name-keyed `models` map collapses the duplicate/colliding model names the AC5.3 validation needs, so the ordered declaration list supplies them). `SourceModel` carries `macro_spec: Option` so `project_macro_registry` is keyed only on the macro-marked models. `collect_module_idents`/`equation_is_module_call` now consult the `MacroRegistry` so a `y = MYMACRO(...)` caller is pre-classified as module-backed (correct `PREVIOUS`/`INIT` rewrite); a macro-body variable's `enclosing_model` is the macro name so a renamed `init`/`previous` builtin inside a like-named macro resolves to the intrinsic (#554). `Diagnostic` includes a `severity` field (`Error`/`Warning`) and `DiagnosticError` variants: `Equation`, `Model`, `Unit`, `Assembly`. The salsa inputs store the `datamodel::*` types directly on their multi-field structs (`SourceProject.sim_specs`/`dimensions`/`units`, `SourceModel.model_sim_specs`, `SourceVariable.equation`/`gf`/`module_refs`) -- per-field salsa read tracking is what gives fine-grained invalidation, so no mirror-projection types are needed; `datamodel_variable_from_source` re-assembles the kind-tagged `datamodel::Variable` for the parser. `datamodel::Equation::Arrayed` carries `has_except_default` to drive EXCEPT default application. `VariableDeps` includes `init_referenced_vars` to track variables referenced by `INIT()` calls. Dependency extraction uses two calls to `classify_dependencies()` (one for the dt AST, one for the init AST) instead of separate walker functions. `parse_source_variable_with_module_context` is the sole parse entry point (the non-module-context variant was removed). `variable_relevant_dimensions` provides dimension-granularity invalidation: scalar variables produce an empty dimension set so dimension changes never invalidate their parse results. `compile_var_fragment` is the salsa-tracked per-variable fragment compiler; its lowering half lives in the sibling `src/db/var_fragment.rs` (`lower_var_fragment`). For a recurrence SCC that `db::dep_graph::resolve_recurrence_sccs` resolved (element-acyclic), the per-element symbolic segments of every member are interleaved into ONE combined `PerVarBytecodes` along the SCC's `element_order` by `combine_scc_fragment` (the per-element-granular generalization of `compiler::symbolic::concatenate_fragments`), built from each member's exact production symbolic fragment via `var_phase_symbolic_fragment_prod(.., scc.phase)` (loud-safe: an unsourceable member or a segmentation error returns `None`/`Err` and the model keeps its `CircularDependency`). `assemble_module` skips each member's per-variable fragment and injects this combined fragment at the first member's runlist slot (the dt fragment into the flow fragments, the init fragment as one synthetic-ident `SymbolicCompiledInitial`); per-write `SymVarRef` identity is preserved, so variable layout offsets are unchanged and each member stays individually addressable. +- **`src/db.rs`** - The module ROOT of the salsa pipeline. After the db-submodule split it holds only the database itself (`SimlinDb` + its `Db` trait, the sync/staged/restore/`current_source_project` methods, the `StdlibModels` cache, the `impl salsa::Database`/`Db`), the `compile_project_incremental()` production entry point, the LTM-glue surface (`LtmSyntheticVar`/`LtmVariablesResult` result types + the `module_link_score_equation` module-link helper (composite reference / ceteris-paribus / signed `black_box_unit_transfer_equation` fallback) called by the per-shape `link_score_equation_text_shaped` (which lives in `src/db/ltm/compile.rs`) -- since Track A the SOLE derivation of a link score's equation text: the former `(from, to)`-keyed `link_score_equation_text` twin was deleted and assembly's standard scalar Bare score now sources the shaped query directly (`compile_ltm_var_fragment`), so the compiled and reported equations are one value and cannot drift; `module_composite_ports` reads the sub-model's `model_ltm_variables` to decide whether a port has a composite, so the same composite reference is used in exhaustive AND discovery mode)/`find_model_output_ports_for_module`/`module_input_pathways_from_edges`/`generate_max_abs_selection` helpers; the last folds a module input port's "strongest pathway" composite through O(1)-sized accumulator helper variables -- inlining the fold into one nested expression doubles the equation text per pathway, which OOMed on macro modules with hundreds of pathways. Since PR #684 a stockless **passthrough** with input→output pathways ALSO emits pathway/composite vars, and since GH #748 the stock-free early return in `model_ltm_variables` fires only when the model is genuinely STATELESS -- no parent-level stocks, no input ports, and no (transitively) stock-carrying module instance (`modules_carry_state`), so a module-only root whose state lives inside a SMOOTH/DELAY or user sub-model is scored instead of silently emitting nothing, and the exhaustive-mode loop-score builder overrides each `x→m` module link with a **per-exit-port pathway selection** (`compute_module_link_overrides` in `src/db/ltm/mod.rs` emits a `$⁚ltm⁚link_score⁚{x}→{m}⁚via⁚{exit}` alias selecting the pathway the loop actually traverses, threaded into `generate_loop_score_variables` as a `(loop_id, link_index)` side-table) because the composite max-abs-selects across ALL pathways and so picks the wrong output port for a multi-output module; `find_model_output_ports` now sorts its result (GH #680) so the parent's pathway recomputation matches the sub-model's emitted `$⁚ltm⁚path⁚{port}⁚{idx}` indices, the residual being that pinned loops pass an empty override map), the `set_project_ltm_enabled`/`set_project_ltm_discovery_mode` setters plus the `LtmEnabledGuard` RAII scope guard (transiently flips `ltm_enabled` and unconditionally restores it on drop; shared by libsimlin's `simlin_project_get_errors` / from-wasm FFIs (GH #466) and `simlin-mcp-core`'s `read_model`/`edit_model` diagnostic passes (GH #662) so the LTM-only advisory harvest behaves identically across every diagnostic-collection surface), and the `#[cfg(test)]` test-module mounts. Everything else moved into per-concern db submodules (`input`/`query`/`sync`/`diagnostic`/`layout`/`var_fragment`/`fragment_compile`/`assemble`/`dep_graph`/`analysis`/`ltm`/...) and is re-exported at the root so the historical `simlin_engine::db::...` surface (and the `use super::*` globs in the root-mounted test modules and `implicit_deps.rs`) keep resolving. `SimlinDb` OWNS its sync state (a private `sync_state: Option` field; the `#[salsa::db]` macro finds `storage` by type, and the field is only mutated via `&mut self` during sync, never during parallel query execution, so no interior mutability is needed), so incrementality is automatic: `db.sync(project) -> SourceProject` threads the prior handles internally (a no-op re-sync stays a salsa cache hit), `db.sync_staged(project) -> (SourceProject, Option)` additionally returns the PRE-staging handles for an exact rollback, `db.restore(project, prev)` re-syncs the prior datamodel with those handles, and `db.current_source_project()` reads the latest handle. A SECOND, `pub(crate)` slot -- `expanded_state: Option`, driven by `db.sync_expanded(expanded_datamodel)` -- holds the conveyor/queue-expanded twin of the project so the special-stock build path is incremental too (see "Conveyors and queues"); it stays `None` until a special-stock model is first built, is never cleared thereafter (salsa cannot reclaim inputs, so clearing would only force a second set to be minted), and its handle is reachable only as `sync_expanded`'s return value, which is what keeps a rolled-back staged patch from poisoning it. `sync_from_datamodel` (fresh `&db` path returning `SyncResult`) and `sync_from_datamodel_incremental` stay public as the internal mechanism the methods wrap and for the read-only test path; `collect_all_diagnostics(db, project: SourceProject)` iterates `project.models(db)`. `SourceProject` carries `ltm_enabled` and `ltm_discovery_mode` flags for LTM compilation, plus `macro_declarations: Vec<(String, Option)>` -- the ordered, pre-dedup list of project models' canonical names + macro specs that the demand-driven `project_macro_registry` query reads to re-derive the macro build error (the canonical-name-keyed `models` map collapses the duplicate/colliding model names the AC5.3 validation needs, so the ordered declaration list supplies them). `SourceModel` carries `macro_spec: Option` so `project_macro_registry` is keyed only on the macro-marked models. `collect_module_idents`/`equation_is_module_call` now consult the `MacroRegistry` so a `y = MYMACRO(...)` caller is pre-classified as module-backed (correct `PREVIOUS`/`INIT` rewrite); a macro-body variable's `enclosing_model` is the macro name so a renamed `init`/`previous` builtin inside a like-named macro resolves to the intrinsic (#554). `Diagnostic` includes a `severity` field (`Error`/`Warning`) and `DiagnosticError` variants: `Equation`, `Model`, `Unit`, `Assembly`. The salsa inputs store the `datamodel::*` types directly on their multi-field structs (`SourceProject.sim_specs`/`dimensions`/`units`, `SourceModel.model_sim_specs`, `SourceVariable.equation`/`gf`/`module_refs`) -- per-field salsa read tracking is what gives fine-grained invalidation, so no mirror-projection types are needed; `datamodel_variable_from_source` re-assembles the kind-tagged `datamodel::Variable` for the parser. `datamodel::Equation::Arrayed` carries `has_except_default` to drive EXCEPT default application. `VariableDeps` includes `init_referenced_vars` to track variables referenced by `INIT()` calls. Dependency extraction uses two calls to `classify_dependencies()` (one for the dt AST, one for the init AST) instead of separate walker functions. `parse_source_variable_with_module_context` is the sole parse entry point (the non-module-context variant was removed). `variable_relevant_dimensions` provides dimension-granularity invalidation: scalar variables produce an empty dimension set so dimension changes never invalidate their parse results. `compile_var_fragment` is the salsa-tracked per-variable fragment compiler; its lowering half lives in the sibling `src/db/var_fragment.rs` (`lower_var_fragment`). For a recurrence SCC that `db::dep_graph::resolve_recurrence_sccs` resolved (element-acyclic), the per-element symbolic segments of every member are interleaved into ONE combined `PerVarBytecodes` along the SCC's `element_order` by `combine_scc_fragment` (the per-element-granular generalization of `compiler::symbolic::concatenate_fragments`), built from each member's exact production symbolic fragment via `var_phase_symbolic_fragment_prod(.., scc.phase)` (loud-safe: an unsourceable member or a segmentation error returns `None`/`Err` and the model keeps its `CircularDependency`). `assemble_module` skips each member's per-variable fragment and injects this combined fragment at the first member's runlist slot (the dt fragment into the flow fragments, the init fragment as one synthetic-ident `SymbolicCompiledInitial`); per-write `SymVarRef` identity is preserved, so variable layout offsets are unchanged and each member stays individually addressable. - **`src/db/input.rs`** (a `db` submodule) - The salsa INPUT layer: the interned key types (`LtmLinkId`/`ModuleIdentContext`/`ModuleInputSet`, the last with its `empty`/`from_canonical_set`/`from_names`/`canonical_input_set` round-trip), the `SourceVariableKind` tag, the three `#[salsa::input]` structs (`SourceProject`/`SourceModel`/`SourceVariable`) that store the synced datamodel field-by-field for fine-grained invalidation, the `PinnedLoopSpec` projection of a non-deleted `LoopMetadata` (a pinned loop's name + canonical variable set, carried on `SourceModel.pinned_loops`; the `LoopMetadata` UIDs are resolved to canonical names at sync time since UIDs never enter the db), the `source_var_is_table_only` lookup-only predicate, `SourceModel.declared_variable_idents` (the ordered, pre-dedup AS-WRITTEN variable-ident list -- the raw data the GH #885 duplicate-canonical-ident check needs, which the canonical-keyed `variables` map collapses; the per-variable analogue of `SourceProject.macro_declarations`), and `datamodel_variable_from_source` (re-assembles the kind-tagged `datamodel::Variable` for the parser). Re-exported at the `db.rs` root. - **`src/db/query.rs`** (a `db` submodule) - The demand-driven read queries over the inputs: per-variable parsing (`parse_source_variable_with_module_context` + `_impl`), the project-global cached contexts (`project_units_context`, `project_datamodel_dims`, `project_dimensions_context`, `project_converted_dimensions`), the module-ident context (`model_module_ident_context`), the per-variable direct-dependency extraction (`variable_direct_dependencies` + `_impl`, the `VariableDeps` value, the `canonical_module_input_set` helper), the implicit-variable metadata (`ImplicitVarMeta`/`model_implicit_var_info`), the recursive `model_module_map`, and the dimension-read queries (`variable_relevant_dimensions`/`variable_dimensions`/`variable_size`). `variable_relevant_dimensions` gives dimension-granularity invalidation (scalars produce an empty set, so dimension changes never invalidate them). - **`src/db/sync.rs`** (a `db` submodule) - Datamodel -> salsa-input sync: the `SyncResult`/`SyncedModel`/`SyncedVariable` handle maps, the `Clone`-able `PersistentSyncState`/`PersistentModelState`/`PersistentVariableState` snapshots threaded between sync calls (with `to_sync_result`/`to_synced_model`/`from_sync_result`), the one-shot stdlib-input builder (`build_stdlib_models`, feeding the root's `StdlibModels` cache), the fresh (`sync_from_datamodel`) and incremental (`sync_from_datamodel_incremental`) entry points + their per-variable helpers (`source_variable_from_datamodel`, `update_source_variable`), the `macro_declarations_from_datamodel` extractor, `pinned_loops_from_datamodel` (resolves a model's `loop_metadata` UIDs to canonical variable names for `SourceModel.pinned_loops`), and `expand_maps_to_chains` (the `maps_to`/`mappings` reachability closure the parser uses to size a variable's dimension dependency). @@ -120,13 +120,14 @@ The primary compilation path uses salsa tracked functions for fine-grained incre - **`src/db/combined_fragment_tests.rs`** - Structural tests for `combine_scc_fragment` (segment ordering along `element_order`, write-`SymVarRef` identity preservation, single trailing `Ret`, per-member resource renumbering with no cross-member collision, merged side-channels). Numeric correctness is the end-to-end job of the `ref.mdl`/`interleaved.mdl` simulation tests. In its own file to keep `db.rs`/`src/db/tests.rs` under the per-file line cap. - **`src/db/ltm/`** - LTM (Loops That Matter) variable generation as salsa tracked functions, split into a directory (one large cohesive concern with heavy internal coupling; a directory keeps the internal helpers `pub(super)`/`pub(crate)` *within* the `ltm` subtree without widening to crate scope, and the names that escape -- the `db.rs`-root `pub use ltm::{...}` / `use ltm::*` surface and the `ltm::` paths `src/db/assemble.rs` uses -- are re-exported at `src/db/ltm/mod.rs`): - **`mod.rs`** - the orchestration root: `model_ltm_variables` (the unified entry point), the entry/glue (`ltm_module_idents`, the submodule `use`/`pub use` wiring, the `#[cfg(test)] #[path]` test mount of the sibling `src/db/ltm_tests.rs` -- which stays one level up from the directory, so the relative mount path is one segment up), and `LtmImplicitVarMeta`/`model_ltm_implicit_var_info`. - - **`parse.rs`** - LTM equation parsing (`parse_ltm_equation` and friends, `reconstruct_ltm_var_lowered`) plus the `datamodel::Equation`-shaping helpers (`ltm_equation_dimensions`, `ltm_synthetic_equation`, `scalarize_ltm_equation`, `retarget_ltm_equation_dims`). + - **`equation.rs`** - `LtmEquation` (+ `LtmArm`), the typed, authoritative representation of an `LtmSyntheticVar`'s equation (GH #965 storage/compile boundary). Mirrors the three `datamodel::Equation` shapes but carries a parsed `Expr0` per arm (plus the generator's exact source-form `text`, kept for the char-dump/diagnostics only), so the fragment compiler and the layout implicit-var scan consume the AST directly -- normal operation never prints an equation to `datamodel::Equation` text and re-parses it back to compile (the former transient-`Aux` round trip, 2-3 parses per equation, GH #655 finding 3). The `Expr0` is `Expr0::new(text)` built once at generation, so this is behavior-neutral: the compiled AST is exactly what the old text parse produced (the char goldens stay byte-identical). Owns the shaping helpers `dimensions`/`scalarize`/`retarget_dims` (they re-tag dimensions and select arms, never rewrite an arm's AST) and `to_flow_ast` (resolves the dimension NAMES to `Dimension`s and yields the `Ast` for lowering). + - **`parse.rs`** - LTM equation parsing (`parse_ltm_equation` and friends, `reconstruct_ltm_var_lowered`): builds the flow-phase `Ast` from the stored `LtmEquation` (no text reparse) and runs the SAME `instantiate_implicit_modules` visitor the ordinary variable parse runs, so PREVIOUS/INIT capture auxes and stdlib module calls expand identically; flow-phase only (LTM vars are flow-phase scalar auxes; the init phase would only re-synthesize same-named helpers that dedup away). Plus the thin free-function wrappers (`ltm_equation_dimensions`, `scalarize_ltm_equation`, `retarget_ltm_equation_dims`) over the `LtmEquation` methods so the call sites keep their spelling. - **`compile.rs`** - per-equation compilation: `compile_ltm_var_fragment`, `link_score_equation_text_shaped`, the shared `compile_ltm_equation_fragment` core, `compile_ltm_synthetic_fragment`, `model_ltm_fragment_diagnostics`, `compile_ltm_implicit_var_fragment`. - **`loops.rs`** - loop assembly: output-port discovery (`find_model_output_ports`), the read-slice / cartesian-subscript helpers (`cartesian_subscripts`, `read_slice_rows`), the tiered loop builders (`build_loops_from_tiered`, `build_element_level_loops`), and the cross-element-through-aggregate recovery cluster (`recover_cross_agg_loops`, `recover_agg_hop_polarities`, `stitch_cross_agg_petals`, `MAX_CROSS_AGG_LOOPS`, `AggLoopBudgetGuard`). - **`link_scores.rs`** - the per-edge / per-shape / through-aggregate link-score emitters (`link_score_dimensions`, `try_cross_dimensional_link_scores`, `try_scalar_to_arrayed_link_scores`, `try_disjoint_dim_arrayed_link_scores`, `emit_per_shape_link_scores`, `emit_source_to_agg_link_scores`, `emit_agg_to_target_link_scores`, `emit_link_scores_for_edge`, `emit_unscoreable_disjoint_edge_warning`), lifted verbatim out of `model_ltm_variables`'s body (they were capture-nothing top-level `fn`s nested in the function). - **`pinned.rs`** - modeler-pinned loops (the `LOOPSCORE` escape hatch, LTM ref section 10). `model_pinned_loops(db, model, project) -> PinnedLoopsResult` reads each non-deleted `SourceModel.pinned_loops` spec (resolved from `LoopMetadata` UIDs to canonical names at sync time), orders its variable set into a closed cycle against `causal_graph_with_modules` (`CausalGraph::order_variable_cycle`, a small Hamiltonian-cycle DFS), validates the cycle contains a stock -- counting stocks INSIDE any module instance the cycle traverses, via the same `enrich_with_module_stocks` enrichment the enumerator applies, so a pin whose only state lives in a SMOOTH/DELAY instance or stock-carrying user sub-model validates while a cycle through a stockless passthrough module is still rejected (GH #673) -- and **dimension-classifies the cycle with the same `classify_cycle` machinery the tiered enumerator uses** (GH #653): a PureScalar pin builds one scalar `Loop`; a PureSameElementA2A pin builds one `Loop` carrying the cycle's `dimensions` + element-level stocks (so its loop score is an arrayed per-element variable); a CrossElementOrMixed pin (literal-element / mixed-shape references -- everything the MDL importer produces) is expanded on the element graph by `expand_pin_on_element_graph` -- project `model_element_causal_edges` onto the pin's variables (+ synthetic agg nodes), guard the subgraph SCC against `MAX_LTM_SCC_NODES`, Johnson, keep circuits whose agg-trimmed variable set equals the pin's, and group them with `build_element_level_loops` -- so a diagonal family collapses into one arrayed `Loop` (with `slot_links` driving per-slot `Equation::Arrayed` scores) and genuinely cross-element instances become element-subscripted scalar `Loop`s. `PinnedLoop.loops: Vec` carries the result with stable pin-derived ids: `pin{n}` for single-loop pins, `pin{n}⁚{j}` for multi-instance ones (never colliding with the enumerator's `r{n}`/`b{n}`/`u{n}` namespace). Invalid pins (unordered set, no stock, oversized expansion SCC, no element-level instantiation) land in `result.invalid` as `(name, reason)` for the caller to surface as a `Warning`. `model_ltm_variables` emits each scored loop's `loop_score` synthetic var + registers its `partition_for_loop` per-slot `loop_partitions` entry in BOTH modes (deduped per scored loop against enumerated loops in exhaustive mode by canonical variable-cycle rotation); in discovery mode the pins are the ONLY loop scores emitted -- this is the whole point. - `model_ltm_variables` generates link scores, loop scores, pathway scores, and composite scores for any model (root, stdlib, user-defined), and also caches the loop-id -> cycle-partition mapping on `LtmVariablesResult::loop_partitions` so post-simulation consumers can normalize loop scores without re-running Tarjan. `LtmVariablesResult::mode` (the `LtmMode { Exhaustive, Discovery }` enum on `db.rs`) carries the resolved loop-enumeration mode out -- `is_discovery` OR'd from the user flag and the variable-level / cross-element SCC auto-flip gates -- so a caller can tell exhaustive Johnson enumeration apart from the discovery heuristic (surfaced through libsimlin's `simlin_sim_get_ltm_mode` -> pysimlin `Run.ltm_mode` / `Sim.get_ltm_mode()`). The discovery decision itself is factored into the shared salsa-tracked `model_ltm_mode` query (this module), so `model_ltm_variables` sets its returned `mode` from it and `db::analysis::model_detected_loops` reads the SAME gate -- the two surfaces can never disagree about exhaustive-vs-discovery. `model_ltm_mode` mirrors the inline gate order exactly (stateless -- stock-free AND no module-internal state, GH #748 -- ⇒ Exhaustive; user flag; cheap variable-level SCC; then, only if those clear, the slow-path cross-element SCC via `model_loop_circuits_tiered`) and accumulates NO diagnostics -- the auto-flip `Warning`s stay in `model_ltm_variables` so they emit once. Relative loop scores are no longer emitted as synthetic variables; see `ltm_post.rs` for the post-simulation computation. Auto-detects sub-model behavior by checking for input ports with causal pathways. Also handles implicit helper/module vars synthesized while parsing LTM equations. `LtmSyntheticVar` carries an `equation: datamodel::Equation` (so an arrayed-per-element-equation target's link score can be a real `Equation::Arrayed` partial-per-element rather than a `"0"` placeholder) plus a `dimensions` field (list of dimension names) so A2A link/loop scores expand to per-element slots during simulation and discovery. Reducer subexpressions are routed through aggregate nodes: `enumerate_agg_nodes` (`ltm_agg.rs`) hoists each statically-describable inlined reducer (whole-extent or sliced) into a synthetic `$⁚ltm⁚agg⁚{n}` aux carrying a `read_slice` / `result_dims`, and `model_ltm_variables` emits that aux plus its two link-score halves -- `source[] → agg` (one scalar `$⁚ltm⁚link_score⁚{from}[]→{agg}`, or `…→{agg}[]` for an arrayed agg, per *read* row -- only the rows the slice reads, scored by the reducer's `classify_reducer` algebraic shortcut over that row's co-reduced slice via `emit_source_to_agg_link_scores`/`read_slice_rows`) and `agg → target` (a Bare partial of `target`'s equation with the reducer subexpr AST-substituted by the agg name; one scalar `$⁚ltm⁚link_score⁚{agg}→{to}[{e}]` per target element for an arrayed `target` -- the agg side carries an `[]` subscript when the agg is itself arrayed, *and* the `Δsource` denominator of that link-score equation projects the same `[]` subscript (`generate_scalar_to_element_equation`'s `source_ref_override`), since the bare multi-slot agg name doesn't compile as a scalar denominator -- or a single `$⁚ltm⁚link_score⁚{agg}→{to}` for a scalar one). An *arrayed* synthetic agg's two halves use the same agg-half emitters with subscripted agg names; the `agg → target` equation BODY pins EVERY arrayed agg ident -- the live agg AND each PREVIOUS-frozen co-agg when the target's equation holds multiple hoisted reducers -- to the target element's PROJECTION onto that agg's own `result_dims` axes (`generate_scalar_to_element_equation`'s per-ident `source_pins` map -- GH #528 for the live agg, GH #751 for the frozen co-aggs, whose bare multi-slot name would not compile as a scalar and previously silent-stubbed the half), so both the diagonal case (`result_dims` equal the target's iterated dims, where the projection is the full tuple) and the strict-prefix *broadcast* case (`SUM(matrix[D1,*])` inside an A2A body over `D1 x D2`, where the full tuple would over-subscript the 1-D agg) compile and score. A positionally-MAPPED sliced reducer (`SUM(matrix[State,*])` over `matrix[Region,D2]` with a positional `State→Region` mapping, GH #534) is hoisted too: the agg is arrayed over the TARGET's iterated dim (`State`) and both halves remap each source row to the slot of its positionally-corresponding target element (`read_slice_rows` via `iterated_axis_slot_elements`); the GH #528 projection composes unchanged since `result_dims` carry the target dim. A reducer over a *dynamic index* (`SUM(pop[idx,*])`), an ELEMENT-mapped sliced reducer the correspondence declines (GH #756; reverse-declared positional pairs are hoisted since GH #757), a StarRange naming a non-subdimension, and the array-valued `RANK` (GH #771) are the carve-outs: not hoisted, so the reference stays on the conservative/Direct path (`DynamicIndex` for the dynamic-index case); a StarRange over a PROPER subdimension *is* hoisted with the subset on its `Reduced` axis (GH #766), so only the subset rows get edges and scores. `emit_per_shape_link_scores` covers the *non*-reducer (Bare / FixedIndex) references -- and diverts `Direct` `PerElement` sites (GH #525) to `emit_per_element_link_scores`' per-(row, full-target-element) scalars; the obsolete per-shape `⁚wildcard`/`⁚dynamic` link-score variants were retired. The exhaustive path consumes the tiered enumerator (`model_loop_circuits_tiered`) via `build_loops_from_tiered`: fast-path circuits (PureScalar / PureSameElementA2A) materialize directly into Loops, and the slow path flows through `build_element_level_loops` (preserved as the slow-path consumer) which groups element-level circuits into A2A loops (shared ID, with dimensions) or element-subscripted cross-element loops. The per-element distinction for cross-dimensional edges is encoded in the link's `from` string (e.g., `"pop[nyc]"`) and the visited element of an A2A target on the link's `to` string (`"mp[boston]"`), not as a separate shape field, so the loop-score equation references the per-element link score that `try_cross_dimensional_link_scores` emits (named `$⁚ltm⁚link_score⁚{from}[{elem}]→{to}` for an arrayed-source → scalar-target reducer edge -- and `$⁚ltm⁚link_score⁚{from}[{d1,d2}]→{to}[{d1}]` for an arrayed-result reducer) and the subscripted-after-quote A2A slot `"$⁚ltm⁚link_score⁚{from}→{to}"[e]` for a cross-element edge that visits one slot of an A2A score. The mirror cross-dimensional case -- a scalar-source → arrayed-target edge -- is handled by the sibling `try_scalar_to_arrayed_link_scores`, which emits one scalar `LtmSyntheticVar` per *target* element, named `$⁚ltm⁚link_score⁚{from}→{to}[{elem}]` (the element rides in the `to` side instead of the `from` side); a single Bare-A2A var would be undiscoverable because the discovery parser would invent a `{from}[{elem}]` node that doesn't match the scalar source's bare node. EXCEPT when `to` IS a variable-backed reduce reading the scalar source as a SCALAR FEEDER (`growth[D1] = SUM(matrix[D1,*] * scale)`, GH #790): that edge is routed FIRST -- gated on the shared `ltm_agg::scalar_feeder_of_variable_backed_agg` decision -- to ONE Bare A2A score `$⁚ltm⁚link_score⁚{from}→{to}` over the agg's `result_dims` -- or over the OWNER's dims for the GH #777 broadcast slice (`share[D9] = SUM(matrix[a,*] * scale)`, `result_dims` empty, owner arrayed: the single scalar reducer value feeds every slot identically, and a `Scalar` emission would reference the bare multi-slot owner and fail assembly) -- via `generate_scalar_feeder_to_agg_equation` (the changed-last feeder convention), identical to how `emit_source_to_agg_link_scores`' scalar-feeder arm scores the SUBEXPRESSION spelling (`0.1 + SUM(matrix[D1,*] * scale)`, synthetic-agg-backed); the per-target-element machinery would otherwise emit `scale → growth[a]`/`[b]` partials referencing the lagged wildcard slice (a GH #541-class uncompilable fragment) and degrade every loop through the feeder to a warned 0-stub. A *disjoint*-dim arrayed→arrayed edge whose target is a per-element-equation (`Ast::Arrayed`) variable referencing the source by literal element subscripts of a dimension disjoint from the target's (`target[D1,D2]` whose `` equations reference `source[m]`, `m ∈ D3`, D3 disjoint from D1/D2) is handled by `try_disjoint_dim_arrayed_link_scores` (called from `emit_link_scores_for_edge` before the `emit_per_shape_link_scores` fallback; since GH #769 it also accepts an `Ast::ApplyToAll` disjoint-dim target when the sites are ALL `FixedIndex` -- `hub[D2] = pop[a1] * 0.05` gets one `Equation::ApplyToAll` score over the target's dims holding `pop[a1]` live instead of the GH #758 loud skip; any non-`FixedIndex` site on an A2A target keeps the skip byte-identically): it reuses the `model_ltm_reference_sites` IR for `(from, to)` (each site's `shape` is `FixedIndex(elems)` for `source[m]`), and emits one `$⁚ltm⁚link_score⁚{from}[{m}]→{to}` per distinct referenced source element -- an `Equation::Arrayed` over `to`'s dims (via the salsa-cached shaped path → `build_arrayed_link_score_equation`), holding `source[m]` live in the slots that reference it and the trivial-zero guard form (`source[m]` frozen at `PREVIOUS`) elsewhere -- not the silent scalarized stand-in the pre-#510 path produced (`link_score_dimensions` returned `[]` for the disjoint edge, so `retarget_ltm_equation_dims` collapsed the per-element `Equation::Arrayed` to the first slot's text). If the target references the source via a *non-literal* index (a `DynamicIndex` site) the edge is not statically scoreable: `emit_unscoreable_disjoint_edge_warning` accumulates a `CompilationDiagnostic` `Warning` naming the edge and *no* link-score variable is emitted (and the caller does not fall through to `emit_per_shape_link_scores`, which would build the misleading scalarized stand-in). A loop running through an inlined reducer traverses `… → from[d] → $⁚ltm⁚agg⁚{n} → to[e] → …` in the un-trimmed loop-score chain, but the synthetic agg nodes are trimmed from each *reported* `Loop`/`FoundLoop` node sequence (like the internal stocks of `DELAY3`/`SMOOTH`). A cross-element feedback loop *through* an inlined reducer visits the (subscript-free, or for an arrayed agg `[]`-subscripted) agg node more than once, so Johnson never emits it directly: `recover_cross_agg_loops` (`src/db/ltm/loops.rs`, called from `build_element_level_loops`) reconstructs it from the agg-touching elementary "petals" (`agg → … → agg`), stitching each pairwise-disjoint petal subset of size ≥2 into ONE canonical loop -- the petals concatenated in priority order (GH #676: every cyclic ordering of a fixed subset traverses the same edge multiset, since each petal contributes the same `agg→head`/internal/`tail→agg` edges regardless of position, so all orderings share one commutative-product `loop_score`; a single representative keeps the loop budget for genuinely-distinct subsets). It is bounded by a deterministic petal priority (fewest internal nodes first, then a stable joined-name tiebreaker -- makes truncation reproducible), a soft per-agg petal cap (`MAX_AGG_PETALS = 8`, bounding the `2^k` subset enumeration), and a model-wide loop-count budget (`MAX_CROSS_AGG_LOOPS = 256`, threaded as `agg_loop_budget` from `build_loops_from_tiered`/`build_element_level_loops`, `#[cfg(test)]`-overridable via `AggLoopBudgetGuard`); clipping sets `LtmVariablesResult.agg_recovery_truncated` and accumulates a `Warning` (mirroring the auto-flip-to-discovery gate). `recover_agg_hop_polarities` then patches the (variable-graph-invisible, hence Unknown) agg hops for monotone reducers (GH #516). The exhaustive loop-iteration path strips the subscript before calling `try_cross_dimensional_link_scores` (which keys `source_vars` by variable name) and dedupes on the stripped key. The auto-flip gate fires on either the variable-level SCC (cheap pre-Johnson check) or the slow-path subgraph SCC (computed inside the tiered enumerator), so pure-A2A models with thousands of element-level circuits no longer get pulled into discovery just because their full element-graph SCC is N times their variable-level SCC. `compile_ltm_synthetic_fragment` is the shared select-and-compile helper (salsa-cached `(from, to)` path vs. direct compilation of the prepared equation) used by both `assemble_module`'s LTM pass and `model_ltm_fragment_diagnostics` -- a salsa-tracked diagnostic pass (driven by `model_all_diagnostics` when `ltm_enabled`, mirroring the auto-flip warning's reachability) that emits a `Warning` for every LTM synthetic variable -- and (GH #741) every LTM implicit helper from `model_ltm_implicit_var_info`, probed via `compile_ltm_implicit_var_fragment` with the empty input set (byte-identical to assembly for the root; for sub-model instances the input set only flips load kinds, never compile success) -- whose fragment fails to compile. Without it `assemble_module` silently drops the failed fragment and the variable reads a constant 0 -- the silent-stubbing path that previously masked a wrong arrayed flow-to-stock link score. The `ltm_enabled` gate keeps this pass off for projects that never requested LTM (so they pay no LTM synthesis cost), but it no longer hides the warnings from FFI callers: GH #466 closed that gap by making `simlin_project_get_errors` transiently re-enable `ltm_enabled` for the diagnostic-accumulator harvest only (via the libsimlin `LtmEnabledGuard`, under the db lock) whenever any simulation on the project was created with `enable_ltm=true`, so both this fragment-failure warning and the auto-flip warning reach `simlin_project_get_errors` / pysimlin `project.get_errors()`. The GH #486 non-Euler rejection is deliberately NOT among them: it rides the `assemble_simulation` hard `Err` (the compile path), not the accumulator, and `simlin_project_get_errors` computes its compile/`vm_error` channel with LTM OFF precisely so that LTM-only rejection cannot masquerade as a project error -- LTM is an analysis overlay, not part of the project's intrinsic compilability (the GH #466 follow-up). + `model_ltm_variables` generates link scores, loop scores, pathway scores, and composite scores for any model (root, stdlib, user-defined), and also caches the loop-id -> cycle-partition mapping on `LtmVariablesResult::loop_partitions` so post-simulation consumers can normalize loop scores without re-running Tarjan. `LtmVariablesResult::mode` (the `LtmMode { Exhaustive, Discovery }` enum on `db.rs`) carries the resolved loop-enumeration mode out -- `is_discovery` OR'd from the user flag and the variable-level / cross-element SCC auto-flip gates -- so a caller can tell exhaustive Johnson enumeration apart from the discovery heuristic (surfaced through libsimlin's `simlin_sim_get_ltm_mode` -> pysimlin `Run.ltm_mode` / `Sim.get_ltm_mode()`). The discovery decision itself is factored into the shared salsa-tracked `model_ltm_mode` query (this module), so `model_ltm_variables` sets its returned `mode` from it and `db::analysis::model_detected_loops` reads the SAME gate -- the two surfaces can never disagree about exhaustive-vs-discovery. `model_ltm_mode` mirrors the inline gate order exactly (stateless -- stock-free AND no module-internal state, GH #748 -- ⇒ Exhaustive; user flag; cheap variable-level SCC; then, only if those clear, the slow-path cross-element SCC via `model_loop_circuits_tiered`) and accumulates NO diagnostics -- the auto-flip `Warning`s stay in `model_ltm_variables` so they emit once. Relative loop scores are no longer emitted as synthetic variables; see `ltm_post.rs` for the post-simulation computation. Auto-detects sub-model behavior by checking for input ports with causal pathways. Also handles implicit helper/module vars synthesized while parsing LTM equations. `LtmSyntheticVar` carries an `equation: LtmEquation` (the typed `Expr0`-per-arm representation, GH #965; the `Arrayed` variant carries a real partial-per-element rather than a `"0"` placeholder for an arrayed-per-element-equation target) plus a `dimensions` field (list of dimension names) so A2A link/loop scores expand to per-element slots during simulation and discovery. The augmentation generators (`ltm_augment.rs`, `module_link_score_equation`) build each equation's text as before and parse it once into `LtmEquation` at that source-format boundary; the fragment compiler consumes the parsed AST directly, so `LtmSyntheticVar` equations are never re-parsed through `datamodel::Equation`. Reducer subexpressions are routed through aggregate nodes: `enumerate_agg_nodes` (`ltm_agg.rs`) hoists each statically-describable inlined reducer (whole-extent or sliced) into a synthetic `$⁚ltm⁚agg⁚{n}` aux carrying a `read_slice` / `result_dims`, and `model_ltm_variables` emits that aux plus its two link-score halves -- `source[] → agg` (one scalar `$⁚ltm⁚link_score⁚{from}[]→{agg}`, or `…→{agg}[]` for an arrayed agg, per *read* row -- only the rows the slice reads, scored by the reducer's `classify_reducer` algebraic shortcut over that row's co-reduced slice via `emit_source_to_agg_link_scores`/`read_slice_rows`) and `agg → target` (a Bare partial of `target`'s equation with the reducer subexpr AST-substituted by the agg name; one scalar `$⁚ltm⁚link_score⁚{agg}→{to}[{e}]` per target element for an arrayed `target` -- the agg side carries an `[]` subscript when the agg is itself arrayed, *and* the `Δsource` denominator of that link-score equation projects the same `[]` subscript (`generate_scalar_to_element_equation`'s `source_ref_override`), since the bare multi-slot agg name doesn't compile as a scalar denominator -- or a single `$⁚ltm⁚link_score⁚{agg}→{to}` for a scalar one). An *arrayed* synthetic agg's two halves use the same agg-half emitters with subscripted agg names; the `agg → target` equation BODY pins EVERY arrayed agg ident -- the live agg AND each PREVIOUS-frozen co-agg when the target's equation holds multiple hoisted reducers -- to the target element's PROJECTION onto that agg's own `result_dims` axes (`generate_scalar_to_element_equation`'s per-ident `source_pins` map -- GH #528 for the live agg, GH #751 for the frozen co-aggs, whose bare multi-slot name would not compile as a scalar and previously silent-stubbed the half), so both the diagonal case (`result_dims` equal the target's iterated dims, where the projection is the full tuple) and the strict-prefix *broadcast* case (`SUM(matrix[D1,*])` inside an A2A body over `D1 x D2`, where the full tuple would over-subscript the 1-D agg) compile and score. A positionally-MAPPED sliced reducer (`SUM(matrix[State,*])` over `matrix[Region,D2]` with a positional `State→Region` mapping, GH #534) is hoisted too: the agg is arrayed over the TARGET's iterated dim (`State`) and both halves remap each source row to the slot of its positionally-corresponding target element (`read_slice_rows` via `iterated_axis_slot_elements`); the GH #528 projection composes unchanged since `result_dims` carry the target dim. A reducer over a *dynamic index* (`SUM(pop[idx,*])`), an ELEMENT-mapped sliced reducer the correspondence declines (GH #756; reverse-declared positional pairs are hoisted since GH #757), a StarRange naming a non-subdimension, and the array-valued `RANK` (GH #771) are the carve-outs: not hoisted, so the reference stays on the conservative/Direct path (`DynamicIndex` for the dynamic-index case); a StarRange over a PROPER subdimension *is* hoisted with the subset on its `Reduced` axis (GH #766), so only the subset rows get edges and scores. `emit_per_shape_link_scores` covers the *non*-reducer (Bare / FixedIndex) references -- and diverts `Direct` `PerElement` sites (GH #525) to `emit_per_element_link_scores`' per-(row, full-target-element) scalars; the obsolete per-shape `⁚wildcard`/`⁚dynamic` link-score variants were retired. The exhaustive path consumes the tiered enumerator (`model_loop_circuits_tiered`) via `build_loops_from_tiered`: fast-path circuits (PureScalar / PureSameElementA2A) materialize directly into Loops, and the slow path flows through `build_element_level_loops` (preserved as the slow-path consumer) which groups element-level circuits into A2A loops (shared ID, with dimensions) or element-subscripted cross-element loops. The per-element distinction for cross-dimensional edges is encoded in the link's `from` string (e.g., `"pop[nyc]"`) and the visited element of an A2A target on the link's `to` string (`"mp[boston]"`), not as a separate shape field, so the loop-score equation references the per-element link score that `try_cross_dimensional_link_scores` emits (named `$⁚ltm⁚link_score⁚{from}[{elem}]→{to}` for an arrayed-source → scalar-target reducer edge -- and `$⁚ltm⁚link_score⁚{from}[{d1,d2}]→{to}[{d1}]` for an arrayed-result reducer) and the subscripted-after-quote A2A slot `"$⁚ltm⁚link_score⁚{from}→{to}"[e]` for a cross-element edge that visits one slot of an A2A score. The mirror cross-dimensional case -- a scalar-source → arrayed-target edge -- is handled by the sibling `try_scalar_to_arrayed_link_scores`, which emits one scalar `LtmSyntheticVar` per *target* element, named `$⁚ltm⁚link_score⁚{from}→{to}[{elem}]` (the element rides in the `to` side instead of the `from` side); a single Bare-A2A var would be undiscoverable because the discovery parser would invent a `{from}[{elem}]` node that doesn't match the scalar source's bare node. EXCEPT when `to` IS a variable-backed reduce reading the scalar source as a SCALAR FEEDER (`growth[D1] = SUM(matrix[D1,*] * scale)`, GH #790): that edge is routed FIRST -- gated on the shared `ltm_agg::scalar_feeder_of_variable_backed_agg` decision -- to ONE Bare A2A score `$⁚ltm⁚link_score⁚{from}→{to}` over the agg's `result_dims` -- or over the OWNER's dims for the GH #777 broadcast slice (`share[D9] = SUM(matrix[a,*] * scale)`, `result_dims` empty, owner arrayed: the single scalar reducer value feeds every slot identically, and a `Scalar` emission would reference the bare multi-slot owner and fail assembly) -- via `generate_scalar_feeder_to_agg_equation` (the changed-last feeder convention), identical to how `emit_source_to_agg_link_scores`' scalar-feeder arm scores the SUBEXPRESSION spelling (`0.1 + SUM(matrix[D1,*] * scale)`, synthetic-agg-backed); the per-target-element machinery would otherwise emit `scale → growth[a]`/`[b]` partials referencing the lagged wildcard slice (a GH #541-class uncompilable fragment) and degrade every loop through the feeder to a warned 0-stub. A *disjoint*-dim arrayed→arrayed edge whose target is a per-element-equation (`Ast::Arrayed`) variable referencing the source by literal element subscripts of a dimension disjoint from the target's (`target[D1,D2]` whose `` equations reference `source[m]`, `m ∈ D3`, D3 disjoint from D1/D2) is handled by `try_disjoint_dim_arrayed_link_scores` (called from `emit_link_scores_for_edge` before the `emit_per_shape_link_scores` fallback; since GH #769 it also accepts an `Ast::ApplyToAll` disjoint-dim target when the sites are ALL `FixedIndex` -- `hub[D2] = pop[a1] * 0.05` gets one `Equation::ApplyToAll` score over the target's dims holding `pop[a1]` live instead of the GH #758 loud skip; any non-`FixedIndex` site on an A2A target keeps the skip byte-identically): it reuses the `model_ltm_reference_sites` IR for `(from, to)` (each site's `shape` is `FixedIndex(elems)` for `source[m]`), and emits one `$⁚ltm⁚link_score⁚{from}[{m}]→{to}` per distinct referenced source element -- an `Equation::Arrayed` over `to`'s dims (via the salsa-cached shaped path → `build_arrayed_link_score_equation`), holding `source[m]` live in the slots that reference it and the trivial-zero guard form (`source[m]` frozen at `PREVIOUS`) elsewhere -- not the silent scalarized stand-in the pre-#510 path produced (`link_score_dimensions` returned `[]` for the disjoint edge, so `retarget_ltm_equation_dims` collapsed the per-element `Equation::Arrayed` to the first slot's text). If the target references the source via a *non-literal* index (a `DynamicIndex` site) the edge is not statically scoreable: `emit_unscoreable_disjoint_edge_warning` accumulates a `CompilationDiagnostic` `Warning` naming the edge and *no* link-score variable is emitted (and the caller does not fall through to `emit_per_shape_link_scores`, which would build the misleading scalarized stand-in). A loop running through an inlined reducer traverses `… → from[d] → $⁚ltm⁚agg⁚{n} → to[e] → …` in the un-trimmed loop-score chain, but the synthetic agg nodes are trimmed from each *reported* `Loop`/`FoundLoop` node sequence (like the internal stocks of `DELAY3`/`SMOOTH`). A cross-element feedback loop *through* an inlined reducer visits the (subscript-free, or for an arrayed agg `[]`-subscripted) agg node more than once, so Johnson never emits it directly: `recover_cross_agg_loops` (`src/db/ltm/loops.rs`, called from `build_element_level_loops`) reconstructs it from the agg-touching elementary "petals" (`agg → … → agg`), stitching each pairwise-disjoint petal subset of size ≥2 into ONE canonical loop -- the petals concatenated in priority order (GH #676: every cyclic ordering of a fixed subset traverses the same edge multiset, since each petal contributes the same `agg→head`/internal/`tail→agg` edges regardless of position, so all orderings share one commutative-product `loop_score`; a single representative keeps the loop budget for genuinely-distinct subsets). It is bounded by a deterministic petal priority (fewest internal nodes first, then a stable joined-name tiebreaker -- makes truncation reproducible), a soft per-agg petal cap (`MAX_AGG_PETALS = 8`, bounding the `2^k` subset enumeration), and a model-wide loop-count budget (`MAX_CROSS_AGG_LOOPS = 256`, threaded as `agg_loop_budget` from `build_loops_from_tiered`/`build_element_level_loops`, `#[cfg(test)]`-overridable via `AggLoopBudgetGuard`); clipping sets `LtmVariablesResult.agg_recovery_truncated` and accumulates a `Warning` (mirroring the auto-flip-to-discovery gate). `recover_agg_hop_polarities` then patches the (variable-graph-invisible, hence Unknown) agg hops for monotone reducers (GH #516). The exhaustive loop-iteration path strips the subscript before calling `try_cross_dimensional_link_scores` (which keys `source_vars` by variable name) and dedupes on the stripped key. The auto-flip gate fires on either the variable-level SCC (cheap pre-Johnson check) or the slow-path subgraph SCC (computed inside the tiered enumerator), so pure-A2A models with thousands of element-level circuits no longer get pulled into discovery just because their full element-graph SCC is N times their variable-level SCC. `compile_ltm_synthetic_fragment` is the shared select-and-compile helper (salsa-cached `(from, to)` path vs. direct compilation of the prepared equation) used by both `assemble_module`'s LTM pass and `model_ltm_fragment_diagnostics` -- a salsa-tracked diagnostic pass (driven by `model_all_diagnostics` when `ltm_enabled`, mirroring the auto-flip warning's reachability) that emits a `Warning` for every LTM synthetic variable -- and (GH #741) every LTM implicit helper from `model_ltm_implicit_var_info`, probed via `compile_ltm_implicit_var_fragment` with the empty input set (byte-identical to assembly for the root; for sub-model instances the input set only flips load kinds, never compile success) -- whose fragment fails to compile. Without it `assemble_module` silently drops the failed fragment and the variable reads a constant 0 -- the silent-stubbing path that previously masked a wrong arrayed flow-to-stock link score. The `ltm_enabled` gate keeps this pass off for projects that never requested LTM (so they pay no LTM synthesis cost), but it no longer hides the warnings from FFI callers: GH #466 closed that gap by making `simlin_project_get_errors` transiently re-enable `ltm_enabled` for the diagnostic-accumulator harvest only (via the libsimlin `LtmEnabledGuard`, under the db lock) whenever any simulation on the project was created with `enable_ltm=true`, so both this fragment-failure warning and the auto-flip warning reach `simlin_project_get_errors` / pysimlin `project.get_errors()`. The GH #486 non-Euler rejection is deliberately NOT among them: it rides the `assemble_simulation` hard `Err` (the compile path), not the accumulator, and `simlin_project_get_errors` computes its compile/`vm_error` channel with LTM OFF precisely so that LTM-only rejection cannot masquerade as a project error -- LTM is an analysis overlay, not part of the project's intrinsic compilability (the GH #466 follow-up). - **`src/db/ltm_tests.rs`** - Unit tests for LTM equation text generation via salsa tracked functions. - **`src/db/ltm_unified_tests.rs`** - Tests for `model_ltm_variables`: simple models, stdlib modules (SMOOTH), passthrough modules, discovery mode. - **`src/db/ltm_module_tests.rs`** - Module-specific LTM tests: SMOOTH compilation with LTM, composite score generation for stdlib modules, user-defined modules with feedback. @@ -177,7 +178,7 @@ The unit subsystem is partial-result throughout: a single bad declaration or one - `tests.rs` - integration-style tests spanning all submodules; preserved as a single file to avoid splitting shared test fixtures. - **`src/ltm_finding.rs`** - Strongest-path loop discovery algorithm (Eberlein & Schoenberg 2020). Post-processes simulation results containing link score synthetic variables to find the most important loops at each timestep. `discover_loops_with_graph(results, causal_graph, stocks, ltm_vars, dims, expansion, sub_model_output_ports, budget)` is the primary entry point: `ltm_vars` and `dims` enable A2A link score expansion (per-element edges from `parse_link_offsets`); when empty, all link scores are treated as scalar. A Bare A2A link score is dimensioned over the TARGET's dims, so `expand_a2a_link_offsets` subscripts the TARGET node per element but PROJECTS the SOURCE node onto `from`'s OWN declared dims -- bare for a scalar feeder (`scale → growth`, GH #790), the same-element diagonal for an equal-dim feeder, the broadcast/partial-collapse form for a lower-dim feeder, and the positionally-mapped diagonal for a `State→Region` pair (GH #527) -- by reusing the element graph's own `db::expand_same_element` rule, so the discovery search graph's node names match `model_element_causal_edges` node-for-node (GH #754; before this, subscripting BOTH endpoints with the score's full tuple minted phantom from-nodes like `scale[a]`/`boost[r,a]`/`x[s]` that named no real node, so every loop through such a feeder dangled and was silently undiscoverable). The from-var declared dims + dimension-mapping context ride in a `LinkExpansionContext` (`declared_dims` + `dim_ctx`) that `analyze_model` builds via the public `analysis::build_link_expansion_context` (the SAME `variable_dimensions` / `project_dimensions_context` queries the element graph reads); the db-less `discover_loops(&Results, &Project)` convenience path passes `LinkExpansionContext::default()` (no A2A expansion runs there). Element-mapped (non-positional) pairs never reach the projection: `db::ltm::link_score_dimensions` declines to retarget them (no Bare A2A score; the GH #758 loud skip fires instead) under the GH #756 positional-only gate. `SearchGraph` provides DFS-based strongest-path traversal from stock nodes. The post-simulation score recompute (path → `FoundLoop`) applies the **per-exit-port pathway selection** (`recompute_module_input_edge_series`, GH #698): a loop edge `x → m` into a module is recomputed by max-abs-selecting over the sub-model's `m·$⁚ltm⁚path⁚{entry}⁚{idx}` pathway scores that end at the exit port the loop reads (recovered from the next link `m → y`), instead of reading the module *composite* (which max-abs-selects across ALL ports and so can pick a wrong-signed port for a multi-output module -- flipping the loop polarity vs exhaustive). The pathway indices come from the module's sub-graph via the same `enumerate_pathways_to_outputs_with_truncation` over the same sorted output-port set the sub-model emitted against, so they match index-for-index (the discovery-mode equivalent of exhaustive's `compute_module_link_overrides`). That port set is NOT re-derived parent-scoped (which would shift indices when another project model reads an extra output port -- GH #698 / PR #705 r3353097150): `discover_loops_with_graph` takes a `SubModelOutputPorts` map (sub-model canonical name -> emitted sorted port set) that `analyze_model` builds from the SAME emission decision via `analysis::build_sub_model_output_ports` -> `db::ltm::sub_model_output_ports` (identity-by-construction); the db-less `discover_loops(&Results, &Project)` convenience path reconstructs the same project-wide-union + stdlib-`output` semantics in `project_sub_model_output_ports`. Falls back to the base composite offset for single-output / pathless / indeterminate-port / sub-model-absent-from-map edges. Because the discovery graph is element-level, the recompute `strip_subscript`s `link.from`/`link.to`/`next.from`/`next.to` before its name matches (arrayed loop nodes carry `[elem]`; mirrors the exhaustive twin's stripping -- PR #705 r3353758167); this is currently latent parity defense, since an arrayed loop through a multi-output module is not yet discoverable end-to-end (a scalar module output feeding an arrayed reader scores a constant-0 link, dropping the loop -- GH #716). Returns a `DiscoveryResult` whose `FoundLoop`s carry per-timestep link/loop/pathway scores, ranked **competitive-first** (`rank_and_filter`): loops sharing their cycle partition with at least one other discovered loop come first, ordered by mean partition-relative importance; loops trivially ALONE in their partition (relative score exactly +/-1 by construction -- zero information, e.g. C-LEARN's isolated gas-uptake decay loops) sort after all competing loops and are dropped first under the `MAX_LOOPS` cap. Each `FoundLoop` carries a result-scoped dense `partition` index into `DiscoveryResult::partitions` (`DiscoveredPartition { stocks, loop_count }`, first-appearance order; NOT stable across runs/edits -- key on the stock set for durable identity), threaded through `analysis::ModelAnalysis::partitions` / `LoopSummary::partition`, the FFI `SimlinDiscoveredPartition` / `SimlinDiscoveredLoop.partition`, and pysimlin `Analysis.partitions` / `Loop.partition`. Also hosts the link-set synthetic-node collapse: `trim_synthetic_aggs_from_loop_links` collapses `$⁚ltm⁚agg⁚{n}` nodes out of a single loop's link *cycle*, and the public `collapse_synthetic_links(Vec)` generalizes that to ALL synthetic/macro/module-internal nodes (`ltm::is_synthetic_node_name`) over an arbitrary link *set* -- each chain `X -> $⁚…internal… -> Y` collapses to one composite edge `X -> Y` with product polarity and the per-timestep max-magnitude path score (the composite link score, LTM ref 6.3/6.4); a purely-internal cycle/source is dropped. `CollapsibleLink { from, to, polarity, score: Option> }` is the abstract shape, so structural-only callers (`score = None`) and LTM-run callers share one impl. libsimlin's `simlin_analyze_get_links(.., include_internal=false)` collapses the `get_links()` set through this; `include_internal=true` returns the raw graph. The `#[cfg(test)]` tests live in the sibling `src/ltm_finding_tests.rs` (mounted via `#[cfg(test)] #[path]`, split out for the per-file line cap). - **`src/ltm_agg.rs`** - Aggregate-node enumeration for LTM. `enumerate_agg_nodes` (salsa-tracked) walks every variable's `Expr2` AST left-to-right depth-first and identifies each maximal array-reducer subexpression (`SUM`/`MEAN`/`MIN`/`MAX`/`STDDEV`/`RANK`/`SIZE`); AST-identical subexpressions (keyed by canonical printed equation text) dedupe to one `AggNode`. Two kinds: **synthetic** (`is_synthetic == true`, the reducer is a sub-expression of a larger equation -- a `$⁚ltm⁚agg⁚{n}` aux is minted) and **variable-backed** (`is_synthetic == false`, the reducer is the entire dt-equation of a scalar/A2A variable like `total_population = SUM(pop[*])` -- the variable itself is the agg; EXCEPT a whole-RHS reducer whose shape the variable-backed machinery cannot express -- a MAPPED iterated axis (GH #534: the name-based link-score path cannot remap, so its `Wildcard` partial would silently stub to 0) or NON-ALIGNED result dims (GH #764 / shape-expressiveness T4: a broadcast over extra owner dims `out[D1,D3] = SUM(matrix[D1,*])` or permuted axes `out[D2,D1] = SUM(cube[D1,D2,*])`, where a per-`(row, slot)` slot cannot name a complete owner element) -- which mints a synthetic agg instead (`variable_backed_shape_is_expressible`, the ONE minting condition), riding the two-half emitters + the GH #528 projection). Each `AggNode` carries a `sources: Vec` -- one entry per source variable, SORTED by canonical name and deduped (salsa cache-equality + emission-order determinism; T2 of the shape-expressiveness design), each with its OWN `read_slice: Vec` (one `AxisRead ∈ {Pinned(elem), Iterated{dim, source_dim}, Reduced{subset}}` per THAT source's axes -- which rows of it the reducer actually reads; a scalar feeder like `scale` in `SUM(pop[*] * scale)` carries an empty slice; `Iterated` carries the (target, source) canonical dim pair, equal for the literal case; `Reduced.subset` is `None` for the full extent or the proper-subdimension element subset for a `SUM(arr[*:Sub])` StarRange, GH #766, decided per axis by `classify_axis_access` -- the single per-axis classifier of the shape-expressiveness design) -- and a `result_dims` (the `Iterated` axes' TARGET dims, datamodel-cased -- empty for a whole-extent or pinned-slice reduce). Under the I1 acceptance (`accept_source_slices`, T5 of the shape-expressiveness design / GH #767) every arrayed CO-SOURCE (`Reduced`-bearing slice) carries the identical *canonical* slice (`AggNode::canonical_read_slice` -- the first `Reduced`-bearing source slice, falling back to the first non-empty for the degenerate no-co-source agg), while an ITERATED-DIM PROJECTION FEEDER -- a source whose slice is all-`Iterated` over exactly the canonical slice's iterated target dims, in order, unmapped (`frac[D1]` in `SUM(matrix[D1,*] * frac[D1])`, per-result-slot constant) -- is accepted with ITS OWN slice (`AggNode::source_is_projection_feeder` is the discriminator); per-source consumers (`emit_agg_routed_edges`, `emit_source_to_agg_link_scores`) read `AggNode::source_read_slice(from)`, and name-keyed consumers use `AggNode::reads_var`. A feeder's link-score half is the per-`(row, slot)` CHANGED-LAST equation (`ltm_augment::generate_iterated_feeder_to_agg_equation`, emitted via `link_scores::iterated_feeder_row_scores` from both the synthetic source-half and `try_cross_dimensional_link_scores`' variable-backed feeder branch): the reducer text pinned to the slot with only the feeder frozen -- the arrayed generalization of the GH #737 scalar-feeder convention, exactly complementary per slot to the co-source rows' changed-first numerators for a bilinear body; the co-source rows' changed-first body partial pins the mismatched-arity feeder dep BY DIM NAME (`pin_body_to_row`'s GH #767 extension) so `PREVIOUS(frac[d1·r])` is held frozen at the row instead of bailing to the delta-ratio fallback. `compute_read_slice` decides hoistability: a whole-extent reduce (`SUM(pop[*])` ⇒ all-`Reduced`), a sliced one (`SUM(pop[NYC,*])` ⇒ `[Pinned(nyc), Reduced]`, `SUM(matrix[D1,*])` over an A2A-`D1` body ⇒ `[Iterated(d1,d1), Reduced]` → an arrayed agg over `D1`), a mixed one (`SUM(matrix3d[D1,NYC,*])` over an A2A-`D1` body ⇒ `[Iterated(d1,d1), Pinned(nyc), Reduced]`), or a positionally-MAPPED sliced one (GH #534: `SUM(matrix[State,*])` over `matrix[Region,D2]` with a positional `State→Region` mapping ⇒ `[Iterated{state, region}, Reduced]`, `result_dims = [State]` -- the agg is arrayed over the TARGET's iterated dim and the three `Iterated`-axis consumers remap each source row to the slot of its positionally-corresponding target element via `iterated_axis_slot_elements`, the preimage inversion of `mapped_element_correspondence`, so the positional-only/element-map gate is inherited) is hoisted; the carve-outs are (a) a reducer over a *dynamic index* (`SUM(pop[idx,*])`, `idx` non-literal ⇒ not statically describable ⇒ not hoisted, reference stays on the conservative path -- `db::ltm_ir` reclassifies it as `DynamicIndex`), (b) an ELEMENT-mapped sliced reducer the correspondence declines (execution resolves positionally and ignores the map, GH #756; a POSITIONAL mapping is accepted in either declaration direction since GH #757) ⇒ `compute_read_slice` returns `None`, conservative -- (c) a multi-source slice combination outside the I1 acceptance -- co-sources with differing slices, one variable read with two different slices (I3b), or a no-`Reduced` source that is not the pure iterated projection (a Pinned-axis mix like `SUM(matrix[D1,*] * w[D1, c1])`, a dim-subset/permuted feeder, or any mapped Iterated axis in a feeder combination) -- `combined_read_slice`/`accept_source_slices` return `None`, (d) a StarRange naming a NON-subdimension of its axis (a mid-edit inconsistency; declined rather than silently widened to the full extent), and (e) `RANK` (GH #771: array-valued, so `reducer_is_hoistable` requires `reducer_collapses_to_scalar` and RANK references stay `Direct` -- a bare arg classifies `Bare`, scored by the GH #742 arrayed-capture path; loops through the rank ORDERING are a documented residual). A multi-source reducer whose arrayed args *agree* (`SUM(a[*] + b[*])`, `a`, `b` over the same dim) mints one agg with one `AggSource` per variable, each carrying the shared canonical slice. `AggNodesResult` exposes `aggs` (first-encounter order), `agg_for_key` (by canonical reducer text), and `aggs_in_var` (which aggs occur in a variable's equation, so the element-graph reroute can ask "which agg of `to` reads `from`?"). A variable-backed reduce with a NON-TRIVIAL statically-describable slice is gated by `variable_backed_reduce_agg` (GH #752, generalized by GH #765 / T3 of the shape-expressiveness design) -- the SAME gate `model_element_causal_edges`' dispatch, `build_element_level_loops`' per-circuit routing, and `try_cross_dimensional_link_scores`' row derivation consume, so edges, loop routing, and scores always cover the identical read rows (all three derive from `read_slice_rows`, invariant I4). Accepted: an ALIGNED partial reduce (`row_sum[D1] = SUM(matrix[D1,*])`, `result_dims` equal to the variable's own dims in order -- Pinned-mixed `outf[D1] = MEAN(cube[D1,x,*])` and subset `out[D1] = MEAN(matrix[D1,*:Sub])` slices included: the divisor is the true read count and unread rows get neither edges nor scores) gets read-slice element edges straight onto the variable's element nodes and per-circuit scalar loops whose both-subscripted `matrix[d1,d2]→row_sum[d1]` links resolve the per-`(row, slot)` scores; a scalar-result Pinned/subset slice on a SCALAR owner (`total = SUM(pop[nyc,*])`, `total = SUM(arr[*:Sub])`) routes the read rows into the bare `to` node with matching per-read-row scores; and the ARRAYED-owner scalar-result Pinned/subset BROADCAST slice (`share[Region] = SUM(pop[nyc,*])`, no `Iterated` axis -- GH #777) fans each read row across the FULL target element set (`emit_agg_routed_edges`' broadcast arm emits `pop[nyc,d2] → share[e]` for every `e`; `emit_broadcast_reduce_link_scores` emits the matching per-(read-row, full-target-element) scalar scores `pop[nyc,d2]→share[e]`, the section-3 `PerElement` rule applied to a variable-backed reducer owner), with loop circuits routed to the per-circuit scalar path via `is_broadcast_reduce_edge` -- the read rows are independent of `to`'s dims, so the RELATED-dim (`share[Region]`) and DISJOINT-dim (`share[D9]`) spellings emit identically. Declined: a pure full-extent variable-backed agg keeps the normal reference walker's edges (the true reads for that shape, inert skip). The GH #764 broadcast/permuted result shapes never reach this gate since T4 -- they mint synthetic aggs at enumeration -- so the gate's Iterated-arm alignment check is defense-in-depth. `scalar_feeder_of_variable_backed_agg` (GH #790) is the scalar sibling of `source_is_projection_feeder`: it composes `variable_backed_reduce_agg` with an empty-slice + genuine-`Reduced`-canonical check to recognize a SCALAR FEEDER of a whole-RHS variable-backed reduce (`scale` in `growth[D1] = SUM(matrix[D1,*] * scale)`), so `try_scalar_to_arrayed_link_scores` can route it to the single Bare A2A changed-last feeder score instead of the uncompilable per-target-element partials. `is_synthetic_agg_name` / `synthetic_agg_name` are the `$⁚ltm⁚agg⁚{n}` name helpers. -- **`src/ltm_augment.rs`** - Equation generators for LTM synthetic variables: `generate_link_score_equation_for_link` (ceteris-paribus link scores; takes `RefShape` and source dimension elements to drive per-shape PREVIOUS wrapping), `generate_loop_score_variables` (emits one `loop_score` per loop as a dimension-shaped `datamodel::Equation`: `Scalar` for scalar loops, `ApplyToAll` for dimensioned loops whose links resolve through Bare A2A names, and per-slot `Equation::Arrayed` for dimensioned loops backed by per-element circuits via `Loop.slot_links` -- GH #653; relative loop scores are computed post-simulation in `ltm_post.rs`), `build_partial_equation_shaped` (AST-based PREVIOUS wrapping that holds matching-shape references live and wraps everything else, via `wrap_non_matching_in_previous` and `classify_expr0_subscript_shape`; arrayed-per-element-equation (`Ast::Arrayed`) targets get one partial per element assembled into an `Equation::Arrayed`), `link_score_var_name` (synthetic name helper: Bare gets the canonical `{from}\u{2192}{to}` form, FixedIndex prepends `[elem]` to from; the obsolete per-shape `\u{205A}wildcard`/`\u{205A}dynamic` Wildcard/DynamicIndex suffixes were retired -- those shapes now collapse onto the Bare name, since *every statically-describable* inlined reducer (whole-extent or sliced) is hoisted into a `$⁚ltm⁚agg⁚{n}` node and only a `DynamicIndex` reference -- `arr[i+1]`, a range, or the not-hoistable dynamic-index reducer carve-out `SUM(pop[idx,*])` -- a whole-RHS variable-backed reducer's `Wildcard` argument, or a de-hoisted array-valued reducer's `Wildcard` arg (`RANK(pop[*], 1)`, GH #771) reach this function), `quote_ident` (identifier quoting for equations). Array support: `classify_reducer` (walks target Expr2 AST to identify reducing builtins -- Linear for SUM/MEAN, Nonlinear for MIN/MAX/STDDEV/RANK, Constant for SIZE -- a thin reader of `ltm_agg::reducer_kind`), `generate_element_to_scalar_equation` (per-element link score equations for arrayed-to-scalar edges, used by both the variable-backed-reducer path and the `source[d] → $⁚ltm⁚agg⁚{n}` half) which dispatches on `ReducerKind` -- `generate_linear_partial` (SUM/MEAN algebraic shortcut), `generate_nonlinear_partial` (MIN/MAX nested binary calls; STDDEV the unrolled population-variance `sqrt` ceteris-paribus partial -- divisor `N`, matching `vm.rs::Opcode::ArrayStddev`, with the mean string-inlined; RANK the documented delta-ratio stand-in pinned by `test_generate_rank_keeps_delta_ratio` -- an order statistic, non-differentiable and unreachable via a real model RHS), `generate_scalar_to_element_equation` (per-element link score for the `$⁚ltm⁚agg⁚{n} → target[e]` half; takes a `source_ref_override: Option<&str>` so a multi-slot arrayed agg's `Δsource` denominator carries the projected `agg[]` subscript instead of the bare agg name, which wouldn't compile as a scalar), `substitute_reducers_in_expr0` (textually replaces a recognized reducer subexpression in an `Expr0` with its agg name, for the `$⁚ltm⁚agg⁚{n} → target` link score), `resolve_link_score_name_for_loop` (picks the Bare-or-FixedIndex link-score name a loop-score reference should target). Module link score formulas (black-box delta-ratio and composite-ref) are inlined directly into `link_score_equation_text` in `db.rs`. The partial-equation builders (`build_partial_equation_shaped`/`_with_live_ref`, `subscript_idents_at_element`) and every link-score equation generator return `Result<_, PartialEquationError>`: a parse failure (genuine `Err`, or an empty `Ok(None)` equation) has no AST to PREVIOUS-wrap, so emitting the unwrapped input would silently produce a non-ceteris-paribus "partial" identical to the full equation (link score magnitude constant |Δz/Δz| = 1) -- a hidden attribution error that compiles cleanly (GH #311). The db-bearing callers (`link_score_equation_text`, `link_score_equation_text_shaped`, the `src/db/ltm/link_scores.rs` emitters) convert the error into a `Warning` (`emit_ltm_partial_equation_warning`, naming the variable + offending equation text) and skip the variable -- distinct from `model_ltm_fragment_diagnostics`, which only catches *compile* failures. The failure is effectively unreachable in production (the text is always a `print_eqn` re-print; an empty equation is rejected as an `EmptyEquation` Error upstream), so this is defense-in-depth. The `#[cfg(test)]` tests live in the sibling `src/ltm_augment_tests.rs` (split out for the per-file line cap). +- **`src/ltm_augment.rs`** - Equation generators for LTM synthetic variables: `generate_link_score_equation_for_link` (ceteris-paribus link scores; takes `RefShape` and source dimension elements to drive per-shape PREVIOUS wrapping), `generate_loop_score_variables` (emits one `loop_score` per loop as a dimension-shaped `datamodel::Equation`: `Scalar` for scalar loops, `ApplyToAll` for dimensioned loops whose links resolve through Bare A2A names, and per-slot `Equation::Arrayed` for dimensioned loops backed by per-element circuits via `Loop.slot_links` -- GH #653; relative loop scores are computed post-simulation in `ltm_post.rs`), `build_partial_equation_shaped` (AST-based PREVIOUS wrapping that holds matching-shape references live and wraps everything else, via `wrap_non_matching_in_previous` and `classify_expr0_subscript_shape`; arrayed-per-element-equation (`Ast::Arrayed`) targets get one partial per element assembled into an `Equation::Arrayed`), `link_score_var_name` (synthetic name helper: Bare gets the canonical `{from}\u{2192}{to}` form, FixedIndex prepends `[elem]` to from; the obsolete per-shape `\u{205A}wildcard`/`\u{205A}dynamic` Wildcard/DynamicIndex suffixes were retired -- those shapes now collapse onto the Bare name, since *every statically-describable* inlined reducer (whole-extent or sliced) is hoisted into a `$⁚ltm⁚agg⁚{n}` node and only a `DynamicIndex` reference -- `arr[i+1]`, a range, or the not-hoistable dynamic-index reducer carve-out `SUM(pop[idx,*])` -- a whole-RHS variable-backed reducer's `Wildcard` argument, or a de-hoisted array-valued reducer's `Wildcard` arg (`RANK(pop[*], 1)`, GH #771) reach this function), `quote_ident` (identifier quoting for equations). Array support: `classify_reducer` (walks target Expr2 AST to identify reducing builtins -- Linear for SUM/MEAN, Nonlinear for MIN/MAX/STDDEV/RANK, Constant for SIZE -- a thin reader of `ltm_agg::reducer_kind`), `generate_element_to_scalar_equation` (per-element link score equations for arrayed-to-scalar edges, used by both the variable-backed-reducer path and the `source[d] → $⁚ltm⁚agg⁚{n}` half) which dispatches on `ReducerKind` -- `generate_linear_partial` (SUM/MEAN algebraic shortcut), `generate_nonlinear_partial` (MIN/MAX nested binary calls; STDDEV the unrolled population-variance `sqrt` ceteris-paribus partial -- divisor `N`, matching `vm.rs::Opcode::ArrayStddev`, with the mean string-inlined; RANK the documented delta-ratio stand-in pinned by `test_generate_rank_keeps_delta_ratio` -- an order statistic, non-differentiable and unreachable via a real model RHS), `generate_scalar_to_element_equation` (per-element link score for the `$⁚ltm⁚agg⁚{n} → target[e]` half; takes a `source_ref_override: Option<&str>` so a multi-slot arrayed agg's `Δsource` denominator carries the projected `agg[]` subscript instead of the bare agg name, which wouldn't compile as a scalar), `substitute_reducers_in_expr0` (textually replaces a recognized reducer subexpression in an `Expr0` with its agg name, for the `$⁚ltm⁚agg⁚{n} → target` link score), `resolve_link_score_name_for_loop` (picks the Bare-or-FixedIndex link-score name a loop-score reference should target). Module link score formulas (black-box delta-ratio and composite-ref) are inlined directly into `module_link_score_equation` in `db.rs` (called by the per-shape `link_score_equation_text_shaped`). The partial-equation builders (`build_partial_equation_shaped`/`_with_live_ref`, `subscript_idents_at_element`) and every link-score equation generator return `Result<_, PartialEquationError>`: a parse failure (genuine `Err`, or an empty `Ok(None)` equation) has no AST to PREVIOUS-wrap, so emitting the unwrapped input would silently produce a non-ceteris-paribus "partial" identical to the full equation (link score magnitude constant |Δz/Δz| = 1) -- a hidden attribution error that compiles cleanly (GH #311). The db-bearing callers (`link_score_equation_text_shaped` and the `src/db/ltm/link_scores.rs` emitters) convert the error into a `Warning` (`emit_ltm_partial_equation_warning`, naming the variable + offending equation text) and skip the variable -- distinct from `model_ltm_fragment_diagnostics`, which only catches *compile* failures. The failure is effectively unreachable in production (the text is always a `print_eqn` re-print; an empty equation is rejected as an `EmptyEquation` Error upstream), so this is defense-in-depth. The `#[cfg(test)]` tests live in the sibling `src/ltm_augment_tests.rs` (split out for the per-file line cap). - **`src/ltm_augment_with_lookup.rs`** - The implicit-WITH-LOOKUP rules for LTM (GH #910), re-exported from `ltm_augment` so every `crate::ltm_augment::*` path is unchanged. A `v = WITH LOOKUP(input, table)` variable is lowered by the compiler to `LOOKUP(v, input)` (`compiler::apply_implicit_with_lookup`), so a link-score partial that RE-EVALUATES such a target's equation is in gf-INPUT units while the guard form's `PREVIOUS(target)` anchor is in gf-OUTPUT units. `is_implicit_with_lookup` carries the coverage doc: every partial is either a **full re-evaluation** (class 1 -- must be wrapped in the gf application) or a **delta-ratio stand-in** (class 2 -- the RANK arm and the nested-arithmetic arm, already in output units, must NEVER be wrapped or a gf output is fed back through the gf). `WithLookupSlotRefs` resolves the target's table reference ONCE per target (`NoGf` / `Shared` / `PerElement`), so a per-element-gf target costs one row-major `SubscriptIterator` walk rather than one per element; an arrayed target's shared table is pinned as `to[1,...]` because a bare arrayed reference resolves each iterated element's own table offset, which the VM reads as NaN past `table_count`. `compose_with_lookup_polarity` (in `src/ltm/polarity.rs`) is the polarity twin, mirroring `apply_implicit_with_lookup`'s placeholder-Positive and zero-point-table rules. The polarity tests live in `src/ltm/with_lookup_tests.rs`, a child module of `ltm::tests`. - **`src/ltm_post.rs`** - Post-simulation relative loop *and link* score computation. `compute_rel_loop_scores(results, loop_partitions)` normalizes each loop's `loop_score` series against the sum of absolute scores within its cycle partition, using SAFEDIV-0 semantics (zero denominator -> zero result). Called after simulation rather than emitted as synthetic equations to avoid O(P^2) equation-text growth on models with dense partitions. `loop_partitions` is an `IndexMap` (re-exported `engine::indexmap`). `compute_rel_loop_scores*` walk its **emission** order rather than re-sorting the loop ids: the partition-sum denominator accumulates `|loop_score|` in that order, and emission order keeps the IEEE-754 (non-associative) sum bit-for-bit identical to the pre-#461 compile-time emitter (GH #468). Emission order is itself deterministic across salsa cache invalidations / processes because `assign_loop_ids` orders loops by a content-derived key (`ltm::graph::loop_id_sort_key`), so it never flaps even though `IndexMap`'s `PartialEq` (salsa cache equality) is order-insensitive. `compute_rel_link_scores(links, step_count)` is the link-level analogue (GH #652): raw link scores divide by the change in the *target*, so they are not comparable across targets and ranking by raw magnitude surfaces numerically-degenerate links (near-constant targets blow up the score). It groups the input `RelLinkInput { to, score }` links by their `to` target and normalizes each link's score by the per-target, per-timestep sum of `|score|` over all that target's *scored* inputs -- a **signed** value in `[-1, 1]` (sign kept like `compute_rel_loop_scores`), with the same `denom_summand` NaN-exclusion / Inf-retention / SAFEDIV-0 semantics. Denominator scope is the scored inputs only: complete in discovery mode (every causal edge scored) but covering just the in-loop subset in exhaustive mode (documented caveat on the fn). libsimlin's `analyze_links_core` calls it over the final (post-synthetic-collapse) link set so the per-target denominator matches the links the caller receives. - **LTM open work**: known LTM bugs and improvements are tracked on GitHub under the `ltm` label; issue #488 is the pinned epic that organises them by area (core algorithm, discovery/post-sim, augmentation, module/array umbrellas). Each open `ltm`-labelled issue carries file:line references and a suggested fix, so a new session can pick a bite-sized piece without re-investigating the subsystem. diff --git a/src/simlin-engine/examples/ltm_helper_census.rs b/src/simlin-engine/examples/ltm_helper_census.rs index d2fc41e71..7997deae3 100644 --- a/src/simlin-engine/examples/ltm_helper_census.rs +++ b/src/simlin-engine/examples/ltm_helper_census.rs @@ -20,7 +20,8 @@ use std::collections::BTreeMap; use salsa::Setter; use simlin_engine::db::{ - SimlinDb, model_ltm_implicit_var_info, model_ltm_variables, sync_from_datamodel_incremental, + LtmEquation, SimlinDb, model_ltm_implicit_var_info, model_ltm_variables, + sync_from_datamodel_incremental, }; use simlin_engine::{canonicalize, open_vensim, open_xmile}; @@ -144,10 +145,10 @@ fn main() { let mut total_sites = 0usize; for v in <m_vars.vars { let texts: Vec<&str> = match &v.equation { - simlin_engine::datamodel::Equation::Scalar(t) => vec![t.as_str()], - simlin_engine::datamodel::Equation::ApplyToAll(_, t) => vec![t.as_str()], - simlin_engine::datamodel::Equation::Arrayed(_, elems, _, _) => { - elems.iter().map(|(_, t, _, _)| t.as_str()).collect() + LtmEquation::Scalar(arm) => vec![arm.text.as_str()], + LtmEquation::ApplyToAll(_, arm) => vec![arm.text.as_str()], + LtmEquation::Arrayed { elements, .. } => { + elements.iter().map(|(_, arm)| arm.text.as_str()).collect() } }; for text in texts { @@ -175,14 +176,11 @@ fn main() { let mut shown = 0; for v in <m_vars.vars { let (variant, texts): (&str, Vec<&str>) = match &v.equation { - simlin_engine::datamodel::Equation::Scalar(t) => ("Scalar", vec![t.as_str()]), - simlin_engine::datamodel::Equation::ApplyToAll(d, t) => { - let _ = d; - ("ApplyToAll", vec![t.as_str()]) - } - simlin_engine::datamodel::Equation::Arrayed(_, elems, _, _) => ( + LtmEquation::Scalar(arm) => ("Scalar", vec![arm.text.as_str()]), + LtmEquation::ApplyToAll(_, arm) => ("ApplyToAll", vec![arm.text.as_str()]), + LtmEquation::Arrayed { elements, .. } => ( "Arrayed", - elems.iter().map(|(_, t, _, _)| t.as_str()).collect(), + elements.iter().map(|(_, arm)| arm.text.as_str()).collect(), ), }; for text in &texts { diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index b2070de6a..d9cb5193e 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -112,9 +112,9 @@ pub use dep_graph::{ModelDepGraphResult, ResolvedScc, SccPhase, model_dependency mod ltm; use ltm::*; pub use ltm::{ - LtmImplicitVarMeta, ShapedLinkScore, compile_ltm_var_fragment, link_score_equation_text_shaped, - model_ltm_implicit_module_refs, model_ltm_implicit_var_info, model_ltm_mode, - model_ltm_var_name_index, model_ltm_variables, + LtmArm, LtmEquation, LtmImplicitVarMeta, ShapedLinkScore, compile_ltm_var_fragment, + link_score_equation_text_shaped, model_ltm_implicit_module_refs, model_ltm_implicit_var_info, + model_ltm_mode, model_ltm_var_name_index, model_ltm_variables, }; // The cross-agg petal-stitching core, shared with `crate::ltm_finding`'s // discovery-mode recovery (GH #696). @@ -384,18 +384,24 @@ impl Db for SimlinDb {} /// A single LTM synthetic variable definition (name + equation). /// -/// `equation` carries its own dimensionality (`Equation::Scalar`, -/// `Equation::ApplyToAll`, or `Equation::Arrayed`). The redundant -/// `dimensions` field is retained because layout sizing (`compute_layout`) -/// and discovery-time offset parsing (`parse_link_offsets`) key off it; -/// every constructor keeps `equation`'s dimension names in lockstep with +/// `equation` is a typed [`LtmEquation`] -- the authoritative, already-parsed +/// `Expr0` AST the fragment compiler and the layout implicit-var scan consume +/// directly. LTM generated equations are internal compiler artifacts, not +/// user-authored source, so normal operation never prints them to +/// `datamodel::Equation` text and re-parses them back to compile (GH #965 +/// storage/compile boundary; the former transient-`Aux` reparse ran 2-3 times +/// per equation, GH #655 finding 3). The equation carries its own +/// dimensionality (`LtmEquation::{Scalar, ApplyToAll, Arrayed}`); the redundant +/// `dimensions` field is retained because layout sizing (`compute_layout`) and +/// discovery-time offset parsing (`parse_link_offsets`) key off it, and every +/// constructor keeps `equation`'s dimension names in lockstep with /// `dimensions`. When `dimensions` is non-empty the variable occupies /// `product(dim_lengths)` layout slots instead of 1. /// /// `compile_directly` forces `assemble_module`'s LTM pass to compile this /// var's `equation` verbatim instead of re-deriving it from the /// `(from, to)`-keyed salsa cache (`compile_ltm_var_fragment` -> -/// `link_score_equation_text`, which always uses `RefShape::Bare`). It is +/// `link_score_equation_text_shaped(.., Bare)`). It is /// set by `emit_per_shape_link_scores` for a scalar link score whose /// underlying reference shape is *not* `Bare` -- a `Wildcard`/`DynamicIndex` /// reference into a scalar target (e.g. `total = arr[idx]`), where the salsa @@ -403,15 +409,15 @@ impl Db for SimlinDb {} /// ceteris-paribus numerator. (Element-subscripted / `$⁚ltm⁚agg⁚{n}` link /// scores already route directly via name checks; setting it for them is harmless.) // -// `equation: datamodel::Equation` blocks deriving `Eq` (the embedded -// `GraphicalFunction` carries `f64` points) and unconditional `Debug` -// (datamodel types only derive `Debug` under `debug-derive`, off in WASM / -// pysimlin). Salsa only needs `PartialEq` for incrementality. +// `equation: LtmEquation` blocks deriving `Eq` (the embedded `Expr0` carries +// `f64` constants) and unconditional `Debug` (its sub-types only derive `Debug` +// under `debug-derive`, off in WASM / pysimlin). Salsa only needs `PartialEq` +// for incrementality. #[cfg_attr(feature = "debug-derive", derive(Debug))] #[derive(Clone, PartialEq, salsa::Update)] pub struct LtmSyntheticVar { pub name: String, - pub equation: datamodel::Equation, + pub equation: LtmEquation, pub dimensions: Vec, pub compile_directly: bool, } @@ -698,13 +704,12 @@ fn module_output_ref_in_document_order( } /// Equation for a module-involved link score (`from` and/or `to` is a -/// module node in the parent causal graph). Shared verbatim by the -/// `(from, to)`-keyed [`link_score_equation_text`] and the per-shape -/// [`crate::db::link_score_equation_text_shaped`] so the two never drift -/// (the shaped twin's `RefShape` does not change a module link's -/// equation: modules are scalar nodes whose composite-reference / -/// ceteris-paribus / unit-transfer formulas don't reach into the target's -/// AST shape). +/// module node in the parent causal graph). Called by the per-shape +/// [`crate::db::link_score_equation_text_shaped`]; a module link's equation +/// is independent of `RefShape` (modules are scalar nodes whose +/// composite-reference / ceteris-paribus / unit-transfer formulas don't +/// reach into the target's AST shape), so the scalar Bare score the compile +/// path reads and the reported score are the same value. /// /// Three cases, each preferring a faithful link score and only falling /// back to the magnitude-1 [`black_box_unit_transfer_equation`] (NOT the @@ -735,7 +740,7 @@ pub(crate) fn module_link_score_equation( to_name: &str, from_var: Option<&crate::variable::Variable>, to_var: &crate::variable::Variable, -) -> Option { +) -> Option { use crate::common::{Canonical, Ident}; let from_ident = Ident::::new(from_name); @@ -808,9 +813,10 @@ pub(crate) fn module_link_score_equation( let equation = if !from_is_module && to_is_module { // variable -> module: the edge feeds one of `to`'s input ports. let crate::variable::Variable::Module { inputs, .. } = to_var else { - return Some(datamodel::Equation::Scalar( - black_box_unit_transfer_equation(from_name, &module_output_ref(to_name)), - )); + return Some(LtmEquation::scalar(black_box_unit_transfer_equation( + from_name, + &module_output_ref(to_name), + ))); }; match inputs.iter().find(|i| i.src == from_ident) { Some(input) => match composite_ref_for_port(to_name, input.dst.as_str()) { @@ -826,9 +832,10 @@ pub(crate) fn module_link_score_equation( // module node rather than the bare name. let from_output = module_output_ref(from_name); let crate::variable::Variable::Module { inputs, .. } = to_var else { - return Some(datamodel::Equation::Scalar( - black_box_unit_transfer_equation(&from_output, &module_output_ref(to_name)), - )); + return Some(LtmEquation::scalar(black_box_unit_transfer_equation( + &from_output, + &module_output_ref(to_name), + ))); }; let matching_input = inputs .iter() @@ -844,58 +851,18 @@ pub(crate) fn module_link_score_equation( // the unit transfer if the reference can't be located. // // Which output to hold live (GH #971): the reference-site IR - // enumerates the module-output composites in DOCUMENT order, so take - // the first that names `from` -- a deterministic pick across - // processes, unlike the former `identifier_set().find()` over a - // per-process-random `HashSet`. The `identifier_set` scan is kept - // only as a defensive fallback for the (unexpected) case where the IR - // recorded no matching `ModuleOutput` occurrence for `to`, preserving - // the historical set of edges that receive a real partial. - let from_output = module_output_ref_in_document_order( - db, model, project, from_name, to_name, - ) - .or_else(|| { - // GH #971 / Track A stage 3 prep: the deterministic IR pick above - // should name the live output for every module->variable edge that - // receives a real partial (the occurrence IR enumerates every - // module-output composite `to` reads, in document order). The - // per-process-random `identifier_set` scan survives ONLY as a - // defensive fallback for the (expected-unreachable) case where the - // IR recorded no `ModuleOutput` occurrence for `to`. Mark LOUDLY - // whenever the scan actually RESCUES (IR missed, scan found one): - // stage 3 needs a fired-or-not signal before it can delete the scan - // with confidence. The rescued ref itself is unchanged -- this only - // warns; it never alters the emitted score. - let rescued = to_var - .ast() - .map(|ast| crate::variable::identifier_set(ast, &[], None)) - .and_then(|deps| { - let prefix = format!("{from_name}\u{00B7}"); - deps.into_iter() - .find(|d| d.as_str().starts_with(&prefix)) - .map(|d| d.to_string()) - }); - if rescued.is_some() { - // In test/debug builds the assertion is the loud marker (the - // scan is meant to be dead, so a fired assert is real signal a - // module-output occurrence is missing from the IR). In release - // builds `debug_assert!` is a no-op, so also emit a warning line - // -- the repo's `eprintln!` idiom for an unexpected internal - // condition (cf. `model.rs`, `dimensions.rs`). - debug_assert!( - false, - "GH #971: module-output IR pick missed `{from_name}\u{00B7}` \ - in `{to_name}`; the identifier_set fallback rescued it. The \ - occurrence IR should enumerate every module-output composite \ - (Track A stage 3 deletes this fallback once it is proven dead)." - ); - eprintln!( - "warning: LTM module-output IR pick missed `{from_name}\u{00B7}` \ - in `{to_name}`; used the identifier_set fallback (GH #971)." - ); - } - rescued - }); + // (`model_ltm_reference_sites`) enumerates every module-output + // composite `to` reads, in DOCUMENT order, so take the first that + // names `from` -- a deterministic pick across processes. This is the + // single source of the live-output choice: the former + // `identifier_set().find()` fallback scan (a per-process-random + // `HashSet` walk kept behind a loud marker while GH #971 proved the IR + // pick dominant) is deleted. A `None` here means the IR recorded no + // module-output occurrence for `to` -- a genuine structural miss that + // falls through to the unit transfer, exactly as an unlocatable + // reference always did. + let from_output = + module_output_ref_in_document_order(db, model, project, from_name, to_name); match from_output { Some(output_ref) => { let output_ident = Ident::::new(&output_ref); @@ -943,124 +910,7 @@ pub(crate) fn module_link_score_equation( } }; - Some(datamodel::Equation::Scalar(equation)) -} - -#[salsa::tracked(returns(ref))] -pub fn link_score_equation_text<'db>( - db: &'db dyn Db, - link_id: LtmLinkId<'db>, - model: SourceModel, - project: SourceProject, -) -> Option { - use crate::common::{Canonical, Ident}; - - let from_name = link_id.link_from(db); - let to_name = link_id.link_to(db); - let from_ident = Ident::::new(from_name); - let to_ident = Ident::::new(to_name); - - let from_var = reconstruct_single_variable(db, model, project, from_name); - let to_var = reconstruct_single_variable(db, model, project, to_name)?; - - let var_name = format!( - "$\u{205A}ltm\u{205A}link_score\u{205A}{}\u{2192}{}", - from_name, to_name - ); - - let from_is_module = from_var.as_ref().is_some_and(|v| v.is_module()); - let to_is_module = to_var.is_module(); - - // Module-involved links: composite reference, ceteris-paribus, or the - // signed unit-transfer fallback, decided by `module_link_score_equation` - // (shared with the per-shape twin so the two never drift). - if from_is_module || to_is_module { - return module_link_score_equation( - db, - model, - project, - from_name, - to_name, - from_var.as_ref(), - &to_var, - ) - .map(|equation| LtmSyntheticVar { - name: var_name, - equation, - dimensions: vec![], - compile_directly: false, - }); - } - - // Standard ceteris-paribus formula for non-module links. - // - // `link_score_equation_text` keys by `(from, to)` only -- no per-shape - // info. The Bare shape, empty `source_dim_elements`, and `None` - // iterated-dim context reproduce the original pre-Phase-3 behavior (the - // GH #511 context is `None`-safe here: this legacy path is only reached - // for scalar-target link scores). Per-shape callers use the `_shaped` fn. - let mut all_vars = HashMap::new(); - if let Some(ref fv) = from_var { - all_vars.insert(from_ident.clone(), fv.clone()); - } - all_vars.insert(to_ident.clone(), to_var.clone()); - // The target's per-occurrence access-shape IR -- the SAME single classifier - // family the shaped twin (`link_score_equation_text_shaped`) threads. It is - // NOT optional: the ceteris-paribus wrap's GH #517 reducer-freeze arm - // (`subtree_has_live_shape`) consults the stream unconditionally, so passing - // an empty stream here (the pre-fix bug) made the wrap freeze a reducer whole - // where the shaped query recursed into it -- deriving a DIFFERENT partial for - // the very same scalar Bare score whenever the live source appears bare inside - // a reducer of the target's equation. Assembly compiles this legacy fragment - // while `model_ltm_variables` reports the shaped one, so the divergence made - // the VM simulate an equation that disagreed with the one reported. Threading - // the real stream single-sources the two. - let ref_sites = crate::db::ltm_ir::model_ltm_reference_sites(db, model, project); - let to_occurrences: &[crate::db::ltm_ir::OccurrenceSite] = ref_sites - .occurrences - .get(to_name) - .map(Vec::as_slice) - .unwrap_or(&[]); - // A `PartialEquationError` here means the target's equation text could - // not be parsed for the ceteris-paribus partial (GH #311). Skip the - // link-score variable and surface a `Warning` instead of emitting a - // silently non-ceteris-paribus score; `model_ltm_fragment_diagnostics` - // never sees this case because the bad equation would compile cleanly. - let equation = match crate::ltm_augment::generate_link_score_equation_for_link( - &from_ident, - &to_ident, - &RefShape::Bare, - &[], - &to_var, - &all_vars, - None, - // Legacy (from, to)-keyed path: no dims context is threaded at - // all, so the GH #526 other-dep check keeps the permissive - // collapse here too. (This path is only consumed for a SCALAR target, - // whose empty iterated-dim space makes the other-dep verdict - // `NotIterated` regardless of `dep_dims`, so omitting it is sound.) - None, - to_occurrences, - ) { - Ok(eqn) => eqn, - Err(err) => { - emit_ltm_partial_equation_warning(db, model, &var_name, &err); - return None; - } - }; - - // This legacy entry always emits a scalar link score. If the generator - // produced an arrayed variant for an arrayed target, collapse it to a - // scalar equation referencing the array vars directly -- the pre-Phase-3 - // behavior this function reproduces. - let equation = ltm::scalarize_ltm_equation(equation); - - Some(LtmSyntheticVar { - name: var_name, - equation, - dimensions: vec![], - compile_directly: false, - }) + Some(LtmEquation::scalar(equation)) } // `link_score_equation_text_shaped` lives in `db/ltm/compile.rs` (where @@ -1134,7 +984,7 @@ fn generate_max_abs_selection( let acc = acc_name(helpers.len()); helpers.push(LtmSyntheticVar { name: acc.clone(), - equation: datamodel::Equation::Scalar(selection), + equation: LtmEquation::scalar(selection), dimensions: vec![], compile_directly: false, }); @@ -1145,7 +995,7 @@ fn generate_max_abs_selection( let final_acc = acc_name(helpers.len()); helpers.push(LtmSyntheticVar { name: final_acc.clone(), - equation: datamodel::Equation::Scalar(selection), + equation: LtmEquation::scalar(selection), dimensions: vec![], compile_directly: false, }); diff --git a/src/simlin-engine/src/db/ltm/compile.rs b/src/simlin-engine/src/db/ltm/compile.rs index fca781d99..37115c3ee 100644 --- a/src/simlin-engine/src/db/ltm/compile.rs +++ b/src/simlin-engine/src/db/ltm/compile.rs @@ -23,16 +23,15 @@ use crate::db::{ Db, LtmLinkId, LtmSyntheticVar, ModelDepGraphResult, ModuleInputSet, RefShape, SourceModel, SourceProject, SourceVariableKind, VarFragmentResult, build_module_inputs, build_stub_variable, build_submodel_metadata, canonical_module_input_set, compute_layout, - extract_tables_from_source_var, link_score_equation_text, model_dependency_graph, - model_implicit_var_info, model_module_ident_context, model_module_map, - parse_source_variable_with_module_context, project_converted_dimensions, - project_datamodel_dims, project_dimensions_context, project_units_context, - reconstruct_single_variable, variable_dimensions, variable_size, + extract_tables_from_source_var, model_dependency_graph, model_implicit_var_info, + model_module_ident_context, model_module_map, parse_source_variable_with_module_context, + project_converted_dimensions, project_datamodel_dims, project_dimensions_context, + project_units_context, reconstruct_single_variable, variable_dimensions, variable_size, }; -use super::parse::{ltm_equation_dimensions, parse_ltm_equation}; +use super::parse::{ltm_equation_dimensions, parse_ltm_equation, scalarize_ltm_equation}; use super::{ - LtmImplicitVarMeta, ltm_module_idents, model_ltm_implicit_module_refs, + LtmEquation, LtmImplicitVarMeta, ltm_module_idents, model_ltm_implicit_module_refs, model_ltm_implicit_var_info, model_ltm_var_name_index, model_ltm_variables, }; @@ -53,6 +52,33 @@ use super::{ /// Parsed LTM equations may synthesize helper auxes for PREVIOUS/INIT /// and may also expand stdlib module calls, so those implicit vars need /// to be handled the same way as in `compile_var_fragment`. +/// +/// The equation is sourced from the per-shape query +/// [`link_score_equation_text_shaped`] with the `Bare` shape -- the SAME query +/// `model_ltm_variables` emits and reports the standard scalar Bare score from +/// -- so the compiled fragment and the reported/serialized equation are +/// single-sourced and cannot drift. (The former `(from, to)`-keyed +/// `link_score_equation_text` query was a second derivation of the same score; +/// it passed an empty dims context and `dim_ctx=None`, which the prior stage +/// had to re-align occurrence-stream-by-occurrence-stream to keep byte-identical +/// -- routing through the shaped twin retires that parallel derivation +/// entirely.) This fragment stays keyed by `(from, to)` for per-link +/// incrementality; the shaped query is keyed by `(from, to, Bare)`, so an edit +/// to an unrelated edge backdates both and this fragment's bytecode is reused. +/// +/// The shaped query returns the equation shaped to the TARGET (an `ApplyToAll` +/// for a Bare read into an arrayed A2A target), but this `(from, to)`-keyed +/// fragment is consumed ONLY by `compile_ltm_synthetic_fragment`'s sub-case (a) +/// -- the dims-empty SCALAR Bare score, which the emission loop reports as +/// `retarget_ltm_equation_dims(shaped_raw, [])` and lays out with one slot. +/// `scalarize_ltm_equation` here is exactly that retarget for the empty-dims +/// case (identical for `Scalar`/`ApplyToAll`/`Arrayed` inputs) and a no-op for a +/// scalar target, so the compiled fragment stays byte-identical to the reported +/// equation. Without it, a scalar feeder read bare beside a hoisted reducer in +/// an arrayed target (`share[D1] = SUM(arr[*] * scale) + scale`) would compile +/// the raw `ApplyToAll` -- writing element offsets past the 1-slot layout -- and +/// abort the whole model with `NotSimulatable`, while the reported score stayed +/// scalar (the compiled-vs-reported drift this single-sourcing exists to kill). #[salsa::tracked(returns(ref))] pub fn compile_ltm_var_fragment( db: &dyn Db, @@ -60,9 +86,14 @@ pub fn compile_ltm_var_fragment( model: SourceModel, project: SourceProject, ) -> Option { - let lsv = link_score_equation_text(db, link_id, model, project).as_ref()?; + let ShapedLinkScore::Scored(lsv) = + link_score_equation_text_shaped(db, link_id, RefShape::Bare, model, project) + else { + return None; + }; - compile_ltm_equation_fragment(db, &lsv.name, &lsv.equation, model, project) + let equation = scalarize_ltm_equation(lsv.equation.clone()); + compile_ltm_equation_fragment(db, &lsv.name, &equation, model, project) } /// Outcome of [`link_score_equation_text_shaped`] for one @@ -111,19 +142,24 @@ pub enum ShapedLinkScore { /// Compute the per-shape link score equation text for a single causal link. /// -/// Sibling of [`crate::db::link_score_equation_text`]. Where the legacy -/// function keys only on `(from, to)` and emits one variable per causal -/// link, this shape-aware variant emits one variable per -/// `(from, to, shape)` tuple. `model_ltm_variables` calls this once per -/// unique shape in the target's AST so per-shape link scores can be -/// ceteris-paribus scored against their actual reference site. +/// This is the sole derivation of a link score's equation text. Keyed on +/// `(from, to, shape)`, it emits one variable per unique shape in the +/// target's AST so per-shape link scores can be ceteris-paribus scored +/// against their actual reference site. `model_ltm_variables` calls it once +/// per unique shape, and the standard scalar `Bare` score's fragment +/// compiler ([`compile_ltm_var_fragment`]) reads the SAME `(from, to, Bare)` +/// result -- so the compiled fragment and the reported/serialized equation +/// are single-sourced and cannot drift. (A former `(from, to)`-keyed +/// `link_score_equation_text` query was a second derivation of the same +/// score; it was deleted in favour of routing every consumer through this +/// query.) /// /// Returns a [`ShapedLinkScore`] (NOT a bare `Option`) so the emission loop /// can distinguish a `PartialEquationError`-driven unscoreable skip from a /// benign missing variable -- see that type's docs and GH #780. /// -/// Module-involved links delegate to the same module formulas as the -/// legacy function (composite reference / black-box delta-ratio). Their +/// Module-involved links delegate to `module_link_score_equation` for the +/// module formulas (composite reference / black-box delta-ratio). Their /// equations are independent of `shape`, but the variable name still /// carries the suffix so the emission loop can keep one entry per /// (from, to, shape) tuple in the `Vec`. @@ -170,10 +206,10 @@ pub fn link_score_equation_text_shaped<'db>( // Module-involved links: shape doesn't change the equation (modules // are scalar nodes in the causal graph; the composite-reference / // ceteris-paribus / unit-transfer formulas don't reach into the AST). - // Delegate to the shared helper so this twin and the (from, to)-keyed - // `link_score_equation_text` stay byte-identical; key the synthetic - // variable by the shape-driven name so the emission loop's per-shape - // map works. A `None` here (a passthrough module with no composite or + // Delegate to the shared `module_link_score_equation` helper, and key + // the synthetic variable by the shape-driven name so the emission + // loop's per-shape map works. A `None` here (a passthrough module with + // no composite or // output port to score -- see `module_link_score_equation`) is a benign // structural skip, NOT an unscoreable edge. if from_is_module || to_is_module { @@ -768,7 +804,7 @@ fn lower_ltm_variable( pub(crate) fn compile_ltm_equation_fragment( db: &dyn Db, var_name: &str, - equation: &datamodel::Equation, + equation: &LtmEquation, model: SourceModel, project: SourceProject, ) -> Option { @@ -776,14 +812,13 @@ pub(crate) fn compile_ltm_equation_fragment( CompiledVarFragment, PerVarBytecodes, ReverseOffsetMap, VariableLayout, }; - // Project-global dims (datamodel form needed by `parse_ltm_equation`) plus - // the canonicalized context + converted dims, all from the salsa-cached - // queries rather than rebuilt per LTM fragment. + // Project-global dims (datamodel form, used to resolve the equation's + // dimension names) plus the canonicalized context + converted dims, all + // from the salsa-cached queries rather than rebuilt per LTM fragment. let dims = project_datamodel_dims(db, project); let dim_context = project_dimensions_context(db, project); let converted_dims = project_converted_dimensions(db, project); - let units_ctx = project_units_context(db, project); let module_idents = ltm_module_idents(db, model, project); let model_var_names = super::ltm_model_var_names(db, model, project); @@ -793,7 +828,6 @@ pub(crate) fn compile_ltm_equation_fragment( var_name, equation, dims, - units_ctx, Some(module_idents), Some(model_var_names), ); @@ -1640,8 +1674,9 @@ pub(crate) fn compile_ltm_equation_fragment( /// Most synthetic equations are compiled verbatim from `ltm_var.equation` /// (`compile_direct`); the one exception is the standard scalar Bare /// `from→to` link score, which routes through the salsa-cached -/// `(from, to)`-keyed `compile_ltm_var_fragment` so an equation edit that -/// does not change the dependency set reuses the cached fragment. +/// `(from, to)`-keyed `compile_ltm_var_fragment` (itself sourced from the +/// per-shape `link_score_equation_text_shaped(.., Bare)` query) so an equation +/// edit that does not change the dependency set reuses the cached fragment. /// /// Returns `None` -- or a `VarFragmentResult` whose `flow_bytecodes` is /// `None` -- when the synthetic equation fails to parse or compile. @@ -1675,8 +1710,8 @@ pub(crate) fn compile_ltm_synthetic_fragment( // Used for everything except the standard scalar Bare `from→to` // link score, which goes through the salsa-cached // `compile_ltm_var_fragment` path below: that path re-derives the - // equation from `link_score_equation_text` (always scalar, Bare; - // per-shape dimensions, element subscripts and reducer + // equation from `link_score_equation_text_shaped(.., Bare)` (always scalar, + // Bare; per-shape dimensions, element subscripts and reducer // substitutions are applied later in `model_ltm_variables`), so // for anything that carries those it would produce the wrong (or // a degenerate) fragment. diff --git a/src/simlin-engine/src/db/ltm/equation.rs b/src/simlin-engine/src/db/ltm/equation.rs new file mode 100644 index 000000000..772da1344 --- /dev/null +++ b/src/simlin-engine/src/db/ltm/equation.rs @@ -0,0 +1,292 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! The typed, authoritative representation of an LTM synthetic variable's +//! equation ([`LtmEquation`]). +//! +//! GH #965 (storage/compile boundary): LTM generated equations are internal +//! compiler artifacts, not user-authored source. This type carries them as a +//! parsed [`Expr0`] AST -- what the fragment compiler and the layout implicit- +//! var scan actually consume -- so normal operation never prints an equation +//! to `datamodel::Equation` text and re-parses it back to compile it (the +//! former `db/ltm/parse.rs::parse_ltm_equation` transient-`Aux` round trip, +//! run 2-3 times per equation; GH #655 finding 3). +//! +//! Each equation arm ([`LtmArm`]) keeps the generator's exact source-form text +//! ALONGSIDE the parsed AST. The text is the DIAGNOSTIC serialization only -- +//! the characterization dump (`db/ltm_char_tests.rs`) and the partial-equation +//! warning read it, and it preserves the source spelling byte-for-byte -- while +//! the AST is the sole compiled representation. They are created together from +//! one generator output (`expr = Expr0::new(text)`) and only ever moved as a +//! unit (`scalarize`/`retarget_dims` re-tag dimensions, never rewrite an arm), +//! so they cannot drift. + +use std::collections::HashMap; + +use crate::ast::{Ast, Expr0}; +use crate::common::{CanonicalElementName, EquationError}; +use crate::datamodel; +use crate::lexer::LexerType; + +/// One equation arm: the authoritative parsed AST plus its diagnostic text. +/// +/// See the module docs for why both are carried. `expr` is `None` for an +/// empty equation (`Expr0::new("")` yields `Ok(None)`, e.g. a discovery-only +/// stub) or the effectively-unreachable case of a generated equation that +/// fails to parse -- both compile to no bytecode, mirroring how the old text +/// path handled an empty/bad `datamodel::Equation`. +#[cfg_attr(feature = "debug-derive", derive(Debug))] +#[derive(Clone, PartialEq, salsa::Update)] +pub struct LtmArm { + /// The generator's exact source-form spelling. Retained ONLY for + /// diagnostics; never re-parsed to compile. + pub text: String, + /// The authoritative compiled AST (`Expr0::new(text)`). + pub expr: Option, +} + +impl LtmArm { + /// Parse `text` into the authoritative AST once, at the generation + /// (source-format) boundary. + pub fn new(text: String) -> Self { + let expr = match Expr0::new(&text, LexerType::Equation) { + Ok(expr) => expr, + Err(_) => { + // A generated LTM equation is always a `print_eqn` re-print or + // a guard-form assembly of already-parsed sub-expressions, so a + // parse failure here means a bug in the augmentation layer, not + // bad user input. Degrade exactly as the old text path did (an + // unparseable equation carried no AST and compiled to no + // bytecode -> the fragment is dropped and + // `model_ltm_fragment_diagnostics` warns) rather than panicking + // -- libsimlin release builds are panic=abort. + debug_assert!(false, "LTM generated equation failed to parse: {text}"); + None + } + }; + Self { text, expr } + } +} + +/// A synthetic LTM variable's equation, mirroring the three +/// `datamodel::Equation` shapes but carrying a parsed [`Expr0`] per arm +/// instead of source text (see the module docs). +/// +/// Dimension NAMES (datamodel casing) are kept as `String`s so the shaping +/// helpers ([`LtmEquation::dimensions`], [`LtmEquation::scalarize`], +/// [`LtmEquation::retarget_dims`]) and layout sizing read them without a +/// `DimensionsContext`; they are resolved to `Dimension`s only when lowering +/// to an `Ast` for compilation ([`LtmEquation::to_flow_ast`]). +#[cfg_attr(feature = "debug-derive", derive(Debug))] +#[derive(Clone, PartialEq, salsa::Update)] +pub enum LtmEquation { + Scalar(LtmArm), + ApplyToAll(Vec, LtmArm), + Arrayed { + dims: Vec, + /// `(raw element subscript, arm)`, order-preserving so + /// [`LtmEquation::scalarize`] can pick the first slot exactly as the + /// old text path did over a `datamodel::Equation::Arrayed`'s `Vec`. + elements: Vec<(String, LtmArm)>, + default: Option, + has_except_default: bool, + }, +} + +impl LtmEquation { + /// A scalar synthetic equation, parsed once from `text`. + pub fn scalar(text: String) -> Self { + LtmEquation::Scalar(LtmArm::new(text)) + } + + /// An apply-to-all synthetic equation over `dims` (datamodel-cased names), + /// parsed once from the shared body `text`. + pub fn apply_to_all(dims: Vec, text: String) -> Self { + LtmEquation::ApplyToAll(dims, LtmArm::new(text)) + } + + /// A per-element (arrayed) synthetic equation. `elements` is + /// `(raw subscript, body text)` in slot order; each body is parsed once. + pub fn arrayed( + dims: Vec, + elements: Vec<(String, String)>, + default: Option, + has_except_default: bool, + ) -> Self { + LtmEquation::Arrayed { + dims, + elements: elements + .into_iter() + .map(|(subscript, text)| (subscript, LtmArm::new(text))) + .collect(), + default: default.map(LtmArm::new), + has_except_default, + } + } + + /// The equation's diagnostic source text concatenated into a single string + /// (the generator's exact spelling): the scalar / apply-to-all formula + /// verbatim, or -- for the per-element (`Arrayed`) variant -- every element + /// formula plus any default joined by newlines. Mirrors + /// [`datamodel::Equation::source_text`]; a convenience for diagnostics and + /// tests that inspect an equation as text without matching its variant. + pub fn source_text(&self) -> String { + match self { + LtmEquation::Scalar(arm) | LtmEquation::ApplyToAll(_, arm) => arm.text.clone(), + LtmEquation::Arrayed { + elements, default, .. + } => { + let mut parts: Vec<&str> = + elements.iter().map(|(_, arm)| arm.text.as_str()).collect(); + if let Some(default_arm) = default { + parts.push(default_arm.text.as_str()); + } + parts.join("\n") + } + } + } + + /// The dimension names the equation carries (datamodel casing), or `&[]` + /// for a scalar one. These are the names whose product gives the + /// variable's layout slot count, kept in lockstep with + /// `LtmSyntheticVar::dimensions` at every construction site. + pub fn dimensions(&self) -> &[String] { + match self { + LtmEquation::Scalar(_) => &[], + LtmEquation::ApplyToAll(dims, _) => dims, + LtmEquation::Arrayed { dims, .. } => dims, + } + } + + /// Reduce the equation to a scalar one, keeping the first slot's arm. + /// + /// Used by the module-involved link-score path + /// ([`crate::db::module_link_score_equation`]), which always emits a scalar + /// variable regardless of the target's dimensionality, and by + /// [`LtmEquation::retarget_dims`] to collapse a degenerate zero-dimension + /// `Arrayed` (the empty-`dims` case) -- a per-element link score with no + /// dimension to index is meaningless, so it falls back to scalar. For an + /// `Arrayed`, the first per-element slot's arm is used. + pub fn scalarize(self) -> Self { + match self { + LtmEquation::Scalar(_) => self, + LtmEquation::ApplyToAll(_, arm) => LtmEquation::Scalar(arm), + LtmEquation::Arrayed { + elements, default, .. + } => { + let arm = elements + .into_iter() + .next() + .map(|(_, arm)| arm) + .or(default) + .unwrap_or_else(|| LtmArm::new("0".to_string())); + LtmEquation::Scalar(arm) + } + } + } + + /// Re-tag the equation so its dimension names match `dims` (the + /// link-score-dimensions policy result the emission loop assigned to + /// `LtmSyntheticVar::dimensions`). Empty `dims` collapses to `Scalar`; + /// non-empty widens a scalar to `ApplyToAll` or re-targets the dimension + /// names of an existing `ApplyToAll`/`Arrayed`, preserving the arm AST(s) + /// verbatim. + pub fn retarget_dims(self, dims: &[String]) -> Self { + match self { + LtmEquation::Scalar(arm) | LtmEquation::ApplyToAll(_, arm) => { + if dims.is_empty() { + LtmEquation::Scalar(arm) + } else { + LtmEquation::ApplyToAll(dims.to_vec(), arm) + } + } + LtmEquation::Arrayed { + dims: orig, + elements, + default, + has_except_default, + } => { + if dims.is_empty() { + // The link-score-dimensions policy assigned no dimensions: + // a zero-dimension `Arrayed` is degenerate (its per-element + // partials have no dimension to index), so collapse to a + // scalar link score -- the pre-existing behavior for such + // edges. + LtmEquation::Arrayed { + dims: orig, + elements, + default, + has_except_default, + } + .scalarize() + } else { + LtmEquation::Arrayed { + dims: dims.to_vec(), + elements, + default, + has_except_default, + } + } + } + } + } + + /// Build the flow-phase `Ast` for compilation and the layout + /// implicit-var scan, resolving the dimension NAMES to `Dimension`s against + /// the project datamodel dims. Mirrors `variable::parse_equation` MINUS the + /// text parse -- the arm ASTs are already parsed -- and returns any + /// dimension-resolution error alongside (so the compiler drops the fragment + /// and the diagnostic pass warns, exactly as a text parse error did). + /// + /// `None` ast is an empty/unparseable equation (no arm expr): it compiles + /// to no bytecode. LTM synthetic variables are flow-phase only, so there is + /// no init-phase ast to build. + pub fn to_flow_ast( + &self, + dimensions: &[datamodel::Dimension], + ) -> (Option>, Vec) { + match self { + LtmEquation::Scalar(arm) => (arm.expr.clone().map(Ast::Scalar), vec![]), + LtmEquation::ApplyToAll(dims, arm) => { + match crate::variable::get_dimensions(dimensions, dims) { + Ok(resolved) => ( + arm.expr.clone().map(|e| Ast::ApplyToAll(resolved, e)), + vec![], + ), + Err(err) => (None, vec![err]), + } + } + LtmEquation::Arrayed { + dims, + elements, + default, + has_except_default, + } => { + // Mirror `parse_equation`'s Arrayed arm: drop element slots with + // no expr (an empty/unparseable body) rather than error. + let map: HashMap = elements + .iter() + .filter_map(|(subscript, arm)| { + arm.expr + .clone() + .map(|e| (CanonicalElementName::from_raw(subscript), e)) + }) + .collect(); + let default_expr = default.as_ref().and_then(|a| a.expr.clone()); + match crate::variable::get_dimensions(dimensions, dims) { + Ok(resolved) => ( + Some(Ast::Arrayed( + resolved, + map, + default_expr, + *has_except_default, + )), + vec![], + ), + Err(err) => (None, vec![err]), + } + } + } + } +} diff --git a/src/simlin-engine/src/db/ltm/link_scores.rs b/src/simlin-engine/src/db/ltm/link_scores.rs index 4dcf338d6..b4e1b10e5 100644 --- a/src/simlin-engine/src/db/ltm/link_scores.rs +++ b/src/simlin-engine/src/db/ltm/link_scores.rs @@ -14,12 +14,12 @@ use std::collections::{HashMap, HashSet}; use crate::common::{Canonical, Ident}; -use crate::datamodel; use crate::db::{ - CompilationDiagnostic, Db, Diagnostic, DiagnosticError, DiagnosticSeverity, LtmLinkId, - LtmSyntheticVar, RefShape, SourceModel, SourceProject, SourceVariable, SourceVariableKind, - project_dimensions_context, reconstruct_single_variable, variable_dimensions, + CompilationDiagnostic, Db, Diagnostic, DiagnosticError, DiagnosticSeverity, LtmEquation, + LtmLinkId, LtmSyntheticVar, RefShape, SourceModel, SourceProject, SourceVariable, + SourceVariableKind, project_dimensions_context, reconstruct_single_variable, + variable_dimensions, }; use super::compile::{ShapedLinkScore, link_score_equation_text_shaped}; @@ -633,7 +633,7 @@ pub(super) fn try_cross_dimensional_link_scores( }; cross_vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per read row // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -759,7 +759,7 @@ pub(super) fn try_cross_dimensional_link_scores( ); cross_vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per element // bracketed name -> routed direct by `assemble_module`'s // element-subscript check; the flag is irrelevant here. @@ -834,7 +834,7 @@ pub(super) fn try_cross_dimensional_link_scores( ); cross_vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per (reduced-elem, result-elem) // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -963,7 +963,7 @@ fn emit_broadcast_reduce_link_scores( ); cross_vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per (read-row, target-element) // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -1140,7 +1140,7 @@ pub(super) fn try_scalar_to_arrayed_link_scores( }; return Some(vec![LtmSyntheticVar { name, - equation: datamodel::Equation::ApplyToAll(equation_dims.clone(), text), + equation: LtmEquation::apply_to_all(equation_dims.clone(), text), dimensions: equation_dims, // The non-empty `dimensions` route this through the A2A // arm of `compile_ltm_synthetic_fragment`, which compiles @@ -1318,7 +1318,7 @@ pub(super) fn try_scalar_to_arrayed_link_scores( }; Some(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per target element // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -2282,8 +2282,9 @@ pub(super) fn emit_per_shape_link_scores( // behavior where such edges produced a scalar link score. lsv.equation = retarget_ltm_equation_dims(lsv.equation, &target_dims); // A non-`Bare` shape carries a partial that the (from, to)- - // keyed salsa compilation path (`link_score_equation_text`, - // always `RefShape::Bare`) cannot reproduce: a + // keyed salsa compilation path (`compile_ltm_var_fragment` -> + // `link_score_equation_text_shaped(.., Bare)`) cannot + // reproduce: a // `Wildcard`/`DynamicIndex` reference into a scalar target // would have its whole subscript wrapped in `PREVIOUS()` and // the ceteris-paribus numerator zeroed. Force `assemble_module` @@ -2560,7 +2561,7 @@ fn emit_per_element_link_scores( ) { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per (row, element) // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -2699,7 +2700,7 @@ fn iterated_feeder_row_scores( ) { Ok(equation) => vars.push(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // scalar -- one variable per read row // bracketed name -> routed direct by `assemble_module`. compile_directly: false, @@ -2787,14 +2788,14 @@ pub(super) fn emit_source_to_agg_link_scores( ) { Ok(text) => { let equation = if agg.result_dims.is_empty() { - datamodel::Equation::Scalar(text) + LtmEquation::scalar(text) } else { // An arrayed agg's feeder score is per-slot: the agg's own // equation text is already ApplyToAll-compatible over // `result_dims` (it is the agg aux's own equation shape), // and the bare agg/feeder references resolve same-element // / broadcast respectively in A2A context. - datamodel::Equation::ApplyToAll(agg.result_dims.clone(), text) + LtmEquation::apply_to_all(agg.result_dims.clone(), text) }; vars.push(LtmSyntheticVar { name, @@ -2854,9 +2855,9 @@ pub(super) fn emit_source_to_agg_link_scores( // would be emitted (silently zeroing the synthetic agg's loop score). // Mirrors the agg-aux emission above. let agg_eqn = if agg.result_dims.is_empty() { - datamodel::Equation::Scalar(agg.equation_text.clone()) + LtmEquation::scalar(agg.equation_text.clone()) } else { - datamodel::Equation::ApplyToAll(agg.result_dims.clone(), agg.equation_text.clone()) + LtmEquation::apply_to_all(agg.result_dims.clone(), agg.equation_text.clone()) }; let Some(agg_var) = reconstruct_ltm_var_lowered(db, &agg.name, &agg_eqn, model, project) else { return; @@ -3016,7 +3017,7 @@ pub(super) fn emit_source_to_agg_link_scores( }; vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], compile_directly: false, }); @@ -3098,7 +3099,7 @@ pub(super) fn emit_source_to_agg_link_scores( }; vars.push(LtmSyntheticVar { name: var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // bracketed name (+ subscripted synthetic agg) -> routed direct. compile_directly: false, @@ -3462,7 +3463,7 @@ pub(super) fn emit_agg_to_target_link_scores( ) { Ok(equation) => vars.push(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // synthetic agg on the `from` side -> routed direct already. compile_directly: false, @@ -3530,7 +3531,7 @@ pub(super) fn emit_agg_to_target_link_scores( ) { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // synthetic agg on `from` + bracketed `to` -> routed direct. compile_directly: false, @@ -3621,7 +3622,7 @@ pub(super) fn emit_agg_to_target_link_scores( match equation { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], // synthetic agg on `from` + bracketed `to` -> routed direct. compile_directly: false, diff --git a/src/simlin-engine/src/db/ltm/loops.rs b/src/simlin-engine/src/db/ltm/loops.rs index 71e53e941..bc68eea15 100644 --- a/src/simlin-engine/src/db/ltm/loops.rs +++ b/src/simlin-engine/src/db/ltm/loops.rs @@ -1807,9 +1807,9 @@ fn source_to_agg_hop_polarity( // not a model variable, so the graph's variable map has no AST for it). // Mirrors `emit_source_to_agg_link_scores`' reconstruction. let agg_eqn = if agg.result_dims.is_empty() { - datamodel::Equation::Scalar(agg.equation_text.clone()) + super::LtmEquation::scalar(agg.equation_text.clone()) } else { - datamodel::Equation::ApplyToAll(agg.result_dims.clone(), agg.equation_text.clone()) + super::LtmEquation::apply_to_all(agg.result_dims.clone(), agg.equation_text.clone()) }; let Some(agg_var) = super::parse::reconstruct_ltm_var_lowered(db, &agg.name, &agg_eqn, model, project) diff --git a/src/simlin-engine/src/db/ltm/mod.rs b/src/simlin-engine/src/db/ltm/mod.rs index 7bba625e7..28a6ace3c 100644 --- a/src/simlin-engine/src/db/ltm/mod.rs +++ b/src/simlin-engine/src/db/ltm/mod.rs @@ -9,7 +9,8 @@ //! emission for a model's synthetic LTM variables; the per-concern work //! lives in submodules: //! -//! * `parse` -- parsing + `datamodel::Equation`-shaping helpers. +//! * `parse` -- typed-equation lowering helpers (implicit-module +//! instantiation over the stored `LtmEquation` ASTs). //! * `compile` -- per-equation compilation to symbolic bytecodes, the //! per-shape link-score equation-text query, and the compile-failure //! diagnostic pass. @@ -27,16 +28,19 @@ use crate::ltm::strip_subscript; use super::{ Db, SourceModel, SourceProject, SourceVariable, SourceVariableKind, compute_layout, - model_causal_edges, model_implicit_var_info, project_datamodel_dims, project_units_context, + model_causal_edges, model_implicit_var_info, project_datamodel_dims, reconstruct_single_variable, }; mod compile; +mod equation; mod link_scores; mod loops; mod parse; mod pinned; +pub use equation::{LtmArm, LtmEquation}; + // Re-export the LTM surface other `db` submodules (and the `db.rs` root's // `use ltm::*` / `pub use ltm::{...}` blocks) reach. The directory keeps the // internal helpers `pub(super)` within the `ltm` subtree; only the names that @@ -514,7 +518,7 @@ fn max_abs_alias_selection( let acc = acc_name(helpers.len()); helpers.push(super::LtmSyntheticVar { name: acc.clone(), - equation: datamodel::Equation::Scalar(selection), + equation: LtmEquation::scalar(selection), dimensions: vec![], // Like the alias, these accumulator names parse as a // `(from, to)` link score, so they must compile verbatim @@ -526,7 +530,7 @@ fn max_abs_alias_selection( let final_acc = acc_name(helpers.len()); helpers.push(super::LtmSyntheticVar { name: final_acc.clone(), - equation: datamodel::Equation::Scalar(selection), + equation: LtmEquation::scalar(selection), dimensions: vec![], compile_directly: true, }); @@ -771,7 +775,7 @@ fn compute_module_link_overrides( .entry(alias_name.clone()) .or_insert_with(|| super::LtmSyntheticVar { name: alias_name.clone(), - equation: datamodel::Equation::Scalar(alias_eqn), + equation: LtmEquation::scalar(alias_eqn), dimensions: vec![], // Compile the prepared equation verbatim: the name parses as // a `(from, to)` link score (`s→m⁚via⁚pos` => from="s", @@ -843,7 +847,6 @@ pub fn model_ltm_implicit_var_info( let ltm_vars = model_ltm_variables(db, model, project); let dims = project_datamodel_dims(db, project); - let units_ctx = project_units_context(db, project); let module_idents = ltm_module_idents(db, model, project); let model_var_names = ltm_model_var_names(db, model, project); @@ -854,7 +857,6 @@ pub fn model_ltm_implicit_var_info( <m_var.name, <m_var.equation, dims, - units_ctx, Some(module_idents), Some(model_var_names), ); @@ -1082,7 +1084,7 @@ pub fn model_ltm_mode(db: &dyn Db, model: SourceModel, project: SourceProject) - /// /// Pathway and composite scores are generated for models with input ports. /// Module-containing loops are no longer filtered out because -/// `link_score_equation_text` now handles module links via composite refs. +/// `module_link_score_equation` handles module links via composite refs. #[salsa::tracked(returns(ref))] pub fn model_ltm_variables( db: &dyn Db, @@ -1301,9 +1303,9 @@ pub fn model_ltm_variables( // `matrix[D1,*]` is exactly "the `D1`-th row, all of axis 2", so this // evaluates correctly as the `Equation::ApplyToAll` body. let equation = if agg.result_dims.is_empty() { - datamodel::Equation::Scalar(agg.equation_text.clone()) + LtmEquation::scalar(agg.equation_text.clone()) } else { - datamodel::Equation::ApplyToAll(agg.result_dims.clone(), agg.equation_text.clone()) + LtmEquation::apply_to_all(agg.result_dims.clone(), agg.equation_text.clone()) }; vars.push(LtmSyntheticVar { name: agg.name.clone(), @@ -2033,7 +2035,7 @@ pub fn model_ltm_variables( pathway_names.push(path_var_name.clone()); vars.push(LtmSyntheticVar { name: path_var_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], compile_directly: false, }); @@ -2053,7 +2055,7 @@ pub fn model_ltm_variables( vars.extend(acc_helpers); vars.push(LtmSyntheticVar { name: composite_name, - equation: datamodel::Equation::Scalar(equation), + equation: LtmEquation::scalar(equation), dimensions: vec![], compile_directly: false, }); diff --git a/src/simlin-engine/src/db/ltm/parse.rs b/src/simlin-engine/src/db/ltm/parse.rs index ea6f08cc5..83a698a57 100644 --- a/src/simlin-engine/src/db/ltm/parse.rs +++ b/src/simlin-engine/src/db/ltm/parse.rs @@ -2,77 +2,97 @@ // Use of this source code is governed by the Apache License, // Version 2.0, that can be found in the LICENSE file. -//! Parsing and `datamodel::Equation`-shaping helpers for LTM synthetic -//! variables. +//! Typed parsing helpers for LTM synthetic variables. //! -//! These functions turn an LTM synthetic variable's equation text into a -//! parsed (and optionally lowered) `Variable`, and convert between the -//! scalar / `ApplyToAll` / `Arrayed` equation variants the rest of the LTM -//! pipeline produces and consumes. +//! LTM synthetic equations are stored as a typed [`LtmEquation`] (a parsed +//! `Expr0` AST plus its diagnostic text), so these helpers do NOT re-parse +//! source text: they build the flow-phase `Ast` from the already-parsed +//! arms (resolving the dimension names) and run the SAME implicit-module / +//! PREVIOUS-INIT-helper visitor (`instantiate_implicit_modules`) the ordinary +//! variable parse runs. This replaces the former `parse_ltm_equation` transient +//! `datamodel::Variable::Aux` round trip -- printing an equation to text and +//! parsing it back to compile it -- which ran 2-3 times per equation (GH #655 +//! finding 3) and coupled LTM compilation to the printer/parser asymmetry class +//! (GH #913). The `datamodel::Equation`-shaping helpers moved onto `LtmEquation` +//! itself (`scalarize`, `retarget_dims`, `dimensions`); the thin wrappers below +//! keep the call sites' free-function spelling. use std::collections::{HashMap, HashSet}; -use crate::canonicalize; +use crate::builtins_visitor::{empty_macro_registry, instantiate_implicit_modules}; use crate::common::{Canonical, Ident}; use crate::datamodel; +use crate::dimensions::DimensionsContext; use crate::db::{Db, ParsedVariableResult, SourceModel, SourceProject}; -use crate::db::{project_datamodel_dims, project_dimensions_context, project_units_context}; +use crate::db::{project_datamodel_dims, project_dimensions_context}; -use super::ltm_module_idents; +use super::{LtmEquation, ltm_module_idents}; -/// Parse an LTM synthetic variable's equation. +/// Parse an LTM synthetic variable's typed equation into a +/// `Variable` plus any implicit helper/module variables the +/// PREVIOUS/INIT and stdlib-module expansion visitor synthesizes. /// -/// Creates a transient `datamodel::Variable::Aux` carrying `equation` -/// verbatim, runs it through `parse_var` (which invokes `BuiltinVisitor` -/// and `instantiate_implicit_modules`), and returns the parsed variable -/// plus any implicit helper/module variables generated while parsing. +/// The equation is already a parsed AST (`LtmEquation`), so this only resolves +/// its dimension names to `Dimension`s (building the flow-phase `Ast`) +/// and runs `instantiate_implicit_modules` -- the exact visitor the ordinary +/// `variable::parse_var_with_module_context` runs -- so PREVIOUS/INIT capture +/// auxes and stdlib module calls expand identically. /// -/// `equation` already carries its own dimensionality: `Equation::Scalar` -/// for scalar LTM vars, `Equation::ApplyToAll` for A2A vars (so the -/// compiler expands the formula across all dimension elements), and -/// `Equation::Arrayed` for per-element link-score equations -- the -/// `datamodel::Equation` -> `Ast` conversion in `variable.rs` produces -/// the matching `Ast` variant for each. +/// Flow-phase only: LTM synthetic variables are scalar auxes (never stocks), +/// compiled in the flow phase, and their init phase would only re-run the +/// visitor over the same arm bodies, synthesizing the *same-named* helpers that +/// dedup away (`model_ltm_implicit_var_info` keys implicit vars by canonical +/// name), so it is skipped. pub(super) fn parse_ltm_equation( var_name: &str, - equation: &datamodel::Equation, + equation: &LtmEquation, dims: &[datamodel::Dimension], - units_ctx: &crate::units::Context, module_idents: Option<&HashSet>>, model_var_names: Option<&HashSet>>, ) -> ParsedVariableResult { - let dm_var = datamodel::Variable::Aux(datamodel::Aux { - ident: canonicalize(var_name).into_owned(), - equation: equation.clone(), - documentation: String::new(), - units: None, - gf: None, - ai_state: None, - uid: None, - compat: datamodel::Compat::default(), - }); + let dimensions_ctx = DimensionsContext::from(dims); + let (flow_ast, mut errors) = equation.to_flow_ast(dims); let mut implicit_vars = Vec::new(); - let variable = crate::variable::parse_var_with_module_context( - dims, - &dm_var, - &mut implicit_vars, - units_ctx, - |mi| Ok(Some(mi.clone())), - module_idents, - // The model's variable-name set: lets PREVIOUS/INIT accept - // non-shadowed bare element subscripts as static (no helper aux) -- - // GH #654. Every LTM parse site must thread the same set so layout - // (model_ltm_implicit_var_info) and compilation agree on which - // helpers exist. - model_var_names, - // LTM synthetic equations (link/loop scores) are engine-generated - // and never contain user macro invocations -> no registry needed. - None, - // ...and are never a macro body, so no #554 enclosing-macro context. - None, - ); + let ast = match flow_ast { + Some(ast) => match instantiate_implicit_modules( + var_name, + ast, + Some(&dimensions_ctx), + module_idents, + model_var_names, + // LTM synthetic equations are engine-generated and never contain + // user macro invocations -> no registry needed; and are never a + // macro body, so no enclosing-macro context (#554). + empty_macro_registry(), + None, + ) { + Ok((ast, mut new_vars)) => { + implicit_vars.append(&mut new_vars); + Some(ast) + } + Err(err) => { + errors.push(err); + None + } + }, + None => None, + }; + + let variable = crate::variable::Variable::Var { + ident: Ident::new(var_name), + ast, + init_ast: None, + eqn: None, + units: None, + tables: vec![], + is_flow: false, + is_table_only: false, + non_negative: false, + errors, + unit_errors: vec![], + }; ParsedVariableResult { variable, @@ -83,18 +103,16 @@ pub(super) fn parse_ltm_equation( pub(super) fn parse_ltm_equation_for_model_with_ids( db: &dyn Db, var_name: &str, - equation: &datamodel::Equation, + equation: &LtmEquation, project: SourceProject, module_idents: &HashSet>, model_var_names: &HashSet>, ) -> ParsedVariableResult { let dims = project_datamodel_dims(db, project); - let units_ctx = project_units_context(db, project); parse_ltm_equation( var_name, equation, dims, - units_ctx, Some(module_idents), Some(model_var_names), ) @@ -108,7 +126,7 @@ pub(super) fn parse_ltm_equation_for_model_with_ids( pub(super) fn reconstruct_ltm_var_lowered( db: &dyn Db, var_name: &str, - equation: &datamodel::Equation, + equation: &LtmEquation, model: SourceModel, project: SourceProject, ) -> Option { @@ -139,83 +157,21 @@ pub(super) fn reconstruct_ltm_var_lowered( Some(crate::model::lower_variable(&scope, &parsed.variable)) } -/// The dimension names an LTM `Equation` carries (datamodel casing), -/// or `&[]` for a scalar one. These are the names whose product gives -/// the variable's layout slot count, and the same names `compute_layout` -/// reads from `LtmSyntheticVar::dimensions` -- the two are kept in sync -/// at every `LtmSyntheticVar` construction site. -pub(super) fn ltm_equation_dimensions(equation: &datamodel::Equation) -> &[String] { - match equation { - datamodel::Equation::Scalar(_) => &[], - datamodel::Equation::ApplyToAll(dims, _) | datamodel::Equation::Arrayed(dims, _, _, _) => { - dims - } - } +/// The dimension names an LTM `LtmEquation` carries (datamodel casing), +/// or `&[]` for a scalar one. Thin wrapper over [`LtmEquation::dimensions`] +/// so the call sites keep their free-function spelling. +pub(super) fn ltm_equation_dimensions(equation: &LtmEquation) -> &[String] { + equation.dimensions() } -/// Reduce an LTM equation to a scalar one, keeping the equation text. -/// Used by the legacy `(from, to)`-keyed link-score path -/// (`link_score_equation_text`), which always emits a scalar variable -/// regardless of the target's dimensionality, and by -/// [`retarget_ltm_equation_dims`] to collapse a degenerate zero-dimension -/// `Arrayed` (the empty-`dims` case) -- a per-element link score with no -/// dimension to index is meaningless, so it falls back to scalar. For an -/// `Equation::Arrayed`, the first per-element slot's text is used (the -/// legacy path predates per-element link scores and is only ever compiled -/// for scalar targets in practice). -pub(crate) fn scalarize_ltm_equation(equation: datamodel::Equation) -> datamodel::Equation { - match equation { - datamodel::Equation::Scalar(_) => equation, - datamodel::Equation::ApplyToAll(_, text) => datamodel::Equation::Scalar(text), - datamodel::Equation::Arrayed(_, elements, default, _) => { - let text = elements - .into_iter() - .next() - .map(|(_, eqn, _, _)| eqn) - .or(default) - .unwrap_or_else(|| "0".to_string()); - datamodel::Equation::Scalar(text) - } - } +/// Reduce an LTM equation to a scalar one. Thin wrapper over +/// [`LtmEquation::scalarize`]. +pub(crate) fn scalarize_ltm_equation(equation: LtmEquation) -> LtmEquation { + equation.scalarize() } -/// Re-tag a link-score `Equation` so its dimension names match `dims` -/// (the link-score-dimensions policy result the emission loop assigned to -/// `LtmSyntheticVar::dimensions`). Empty `dims` collapses the equation to -/// `Scalar` (via [`scalarize_ltm_equation`] for an `Arrayed` input, since -/// a zero-dimension `Arrayed` is degenerate); non-empty `dims` widens a -/// scalar to `ApplyToAll` or re-targets the dimension names of an existing -/// `ApplyToAll`/`Arrayed`, preserving the equation text / per-element -/// formulas verbatim. -pub(super) fn retarget_ltm_equation_dims( - equation: datamodel::Equation, - dims: &[String], -) -> datamodel::Equation { - use datamodel::Equation::{ApplyToAll, Arrayed, Scalar}; - match equation { - Scalar(text) | ApplyToAll(_, text) => { - if dims.is_empty() { - Scalar(text) - } else { - ApplyToAll(dims.to_vec(), text) - } - } - Arrayed(orig_dims, elements, default, apply_default) => { - if dims.is_empty() { - // The link-score-dimensions policy assigned no dimensions: - // the (rare) arrayed-target edge whose source dimensions - // are incompatible with the target's (so the per-shape A2A - // path was skipped and the cross-dimensional path declined - // it for having a non-scalar target). A zero-dimension - // `Arrayed` is degenerate -- its per-element partials are - // meaningless without a target dimension to index -- so - // collapse to a scalar link score, matching the contract - // in this function's doc comment and the pre-existing - // behavior where such edges produced a scalar link score. - scalarize_ltm_equation(Arrayed(orig_dims, elements, default, apply_default)) - } else { - Arrayed(dims.to_vec(), elements, default, apply_default) - } - } - } +/// Re-tag a link-score `LtmEquation`'s dimension names to `dims`. Thin +/// wrapper over [`LtmEquation::retarget_dims`]. +pub(super) fn retarget_ltm_equation_dims(equation: LtmEquation, dims: &[String]) -> LtmEquation { + equation.retarget_dims(dims) } diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index e7d8387d4..c21761174 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -24,26 +24,37 @@ use crate::datamodel; use crate::test_common::TestProject; /// Render one synthetic variable's equation as a stable, fully-structured -/// string: `Scalar`/`ApplyToAll` are their text; `Arrayed` lists each -/// `element => eqn` slot (element-sorted, as the generator emits) plus the +/// string: `Scalar`/`ApplyToAll` are their diagnostic text; `Arrayed` lists +/// each `element => eqn` slot (element-sorted, as the generator emits) plus the /// optional `default => eqn`. Dimensions are prefixed so a shape change is /// visible in the pin. -fn render_equation(eq: &datamodel::Equation) -> String { +/// +/// Renders the arm's diagnostic `text` (the generator's exact source-form +/// spelling), NOT `print_eqn` of the parsed AST, so the golden pins the same +/// bytes the augmentation layer produced -- the AST is the compiled form, the +/// text the diagnostic serialization (see [`crate::db::LtmEquation`]). +fn render_equation(eq: &crate::db::LtmEquation) -> String { + use crate::db::LtmEquation; match eq { - datamodel::Equation::Scalar(s) => format!("scalar: {s}"), - datamodel::Equation::ApplyToAll(dims, s) => { - format!("a2a[{}]: {s}", dims.join(",")) + LtmEquation::Scalar(arm) => format!("scalar: {}", arm.text), + LtmEquation::ApplyToAll(dims, arm) => { + format!("a2a[{}]: {}", dims.join(","), arm.text) } - datamodel::Equation::Arrayed(dims, elements, default, apply_default) => { + LtmEquation::Arrayed { + dims, + elements, + default, + has_except_default, + } => { let mut out = format!( - "arrayed[{}] (apply_default={apply_default}):", + "arrayed[{}] (apply_default={has_except_default}):", dims.join(",") ); - for (elem, eqn, _, _) in elements { - out.push_str(&format!("\n {elem} => {eqn}")); + for (elem, arm) in elements { + out.push_str(&format!("\n {elem} => {}", arm.text)); } - if let Some(default_eqn) = default { - out.push_str(&format!("\n => {default_eqn}")); + if let Some(default_arm) = default { + out.push_str(&format!("\n => {}", default_arm.text)); } out } @@ -1283,21 +1294,21 @@ fn char_already_lagged_other_dep() { // // `total = scale + SUM(arr[*] * scale)`: `scale` appears bare (outside the // reducer) AND as the reducer's scalar feeder inside `SUM(arr[*] * scale)`. The -// `scale -> total` Bare link score is the finding-1 probe. With an empty -// occurrence stream the LEGACY `link_score_equation_text` (which assembly -// compiles) froze the whole `SUM(...)` reducer -- changed-FIRST -- while the -// SHAPED emitter (which `model_ltm_variables` reports/serializes) threaded the -// real stream and recursed into it -- changed-LAST. So the COMPILED fragment -// disagreed with the REPORTED equation: the VM simulated one derivation and the -// report showed another. Threading the real stream makes the legacy query -// changed-LAST too. This golden pins the emitted (changed-LAST) text -- the -// numerator subtracts `PREVIOUS(scale) + sum(arr[*] * PREVIOUS(scale))` from the -// live `total`, i.e. `scale` is held live in BOTH occurrences -- and -// `legacy_and_shaped_bare_score_agree_when_source_bare_in_reducer` (in -// `db::ltm_tests`) pins that the legacy (compiled) query now returns the -// identical bytes. GH #517/#743: the changed-LAST convention is the correct one -// (freeze the target's OTHER inputs, hold the scored source live everywhere it -// appears), so the drift is resolved onto the shaped semantics, not HEAD's. +// `scale -> total` Bare link score is the finding-1 probe. Historically the +// LEGACY `(from, to)`-keyed `link_score_equation_text` query (which assembly +// compiled) passed an empty occurrence stream and froze the whole `SUM(...)` +// reducer -- changed-FIRST -- while the SHAPED emitter (which +// `model_ltm_variables` reports/serializes) threaded the real stream and +// recursed into it -- changed-LAST. Stage 2b re-aligned the legacy query onto +// the shaped derivation; Track A3 stage 3a then DELETED the legacy query +// outright: assembly's sub-case (a) now sources from +// `link_score_equation_text_shaped(.., Bare)` directly, so the compiled and +// reported equations are one value and cannot drift. This golden pins the +// emitted (changed-LAST) text -- the numerator subtracts +// `PREVIOUS(scale) + sum(arr[*] * PREVIOUS(scale))` from the live `total`, i.e. +// `scale` is held live in BOTH occurrences. GH #517/#743: the changed-LAST +// convention is the correct one (freeze the target's OTHER inputs, hold the +// scored source live everywhere it appears). // --------------------------------------------------------------------------- fn scalar_feeder_bare_in_hoisted_reducer_model() -> datamodel::Project { @@ -1324,3 +1335,97 @@ fn char_scalar_feeder_bare_in_hoisted_reducer() { ); assert_golden("scalar_feeder_bare_in_hoisted_reducer", &actual); } + +// --------------------------------------------------------------------------- +// Model H2 (Track A3 stage 3a, finding 1): the ARRAYED-target sibling of +// Model H -- a SCALAR feeder read BARE alongside a HOISTED reducer, but now +// inside an A2A (arrayed) target rather than a scalar one. +// +// `share[D1] = SUM(arr[*] * scale) + scale`: the `+ scale` is a Bare read of +// the scalar `scale` in an A2A-over-`D1` body, so `scale -> share` has a Bare +// site whose ceteris-paribus partial the shaped query builds as an +// `ApplyToAll(["D1"], ..)` (the target is arrayed). But `link_score_dimensions` +// returns `[]` for a scalar-source -> arrayed-target edge (the feeder carries +// no dimensions to inherit), so `emit_per_shape_link_scores` REPORTS the score +// as a dims-empty SCALAR var (`retarget_ltm_equation_dims(.., [])` collapses the +// A2A equation to `Scalar`) laid out with one slot, and the assembly fragment +// compiler routes it through sub-case (a) -> the salsa-cached +// `compile_ltm_var_fragment`. +// +// The regression this pins: when `compile_ltm_var_fragment` compiled the shaped +// Bare query's RAW `ApplyToAll` equation (writing element offsets 0 AND 1 into +// a 1-slot var) while the emission loop scalarized the same score, the compiled +// fragment disagreed with the reported var and `compile_project_incremental` +// hard-failed with `NotSimulatable` ("element_offset 1 out of bounds"). Sourcing +// the fragment from `scalarize_ltm_equation(shaped_raw)` -- identical to the +// reported `retarget_ltm_equation_dims(shaped_raw, [])` for the dims-empty +// sub-case (a) var, and a no-op for a scalar target -- keeps compiled == reported +// by construction, so the score degrades gracefully (the scalarized A2A text +// references `arr[*]` in scalar context, so it warn-stubs to constant 0, exactly +// as the reported var does) instead of aborting the whole model's compilation. +// --------------------------------------------------------------------------- + +fn scalar_feeder_bare_in_arrayed_reducer_model() -> datamodel::Project { + TestProject::new("scalar_bare_in_arrayed_reducer_char") + .with_sim_time(0.0, 10.0, 1.0) + .named_dimension("D1", &["a", "b"]) + .array_aux("arr[D1]", "1") + .aux("scale", "pop * 0.01", None) + .array_aux("share[D1]", "SUM(arr[*] * scale) + scale") + .flow("growth", "SUM(share[*]) * 0.001", None) + .stock("pop", "1", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn scalar_feeder_bare_in_arrayed_reducer_compiles_and_simulates() { + use salsa::Setter; + + let project = scalar_feeder_bare_in_arrayed_reducer_model(); + let mut db = SimlinDb::default(); + let (source_project, _source_model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + source_project.set_ltm_enabled(&mut db).to(true); + + // The core regression: a scalar feeder read bare beside a hoisted reducer in + // an arrayed A2A target must NOT turn LTM compilation into a hard failure. + // Pre-fix this `.expect` panicked on `NotSimulatable` ("element_offset 1 out + // of bounds for variable $⁚ltm⁚link_score⁚scale→share (size 1)"). + let compiled = compile_project_incremental(&db, source_project, "main") + .expect("LTM incremental compilation should succeed for an arrayed target"); + + // The dims-empty scalar `scale -> share` score exists in the layout with a + // single slot -- the shape the emission loop reports. + let share_score_offsets: Vec = compiled + .offsets + .iter() + .filter(|(k, _)| k.as_str() == "$\u{205A}ltm\u{205A}link_score\u{205A}scale\u{2192}share") + .map(|(_, v)| *v) + .collect(); + assert_eq!( + share_score_offsets.len(), + 1, + "expected exactly one scalar scale->share link score slot" + ); + + // Simulation runs to completion and the model's OWN variables are + // unperturbed by the degraded score. + let mut vm = crate::vm::Vm::new(compiled.clone()).expect("VM creation should succeed"); + vm.run_to_end() + .expect("simulation should run to completion"); + let results = vm.into_results(); + for (key, off) in &compiled.offsets { + if key.as_str().starts_with("$\u{205A}ltm") { + continue; + } + for row in results.iter() { + assert!( + row[*off].is_finite(), + "model variable {key} must stay finite; the LTM overlay must not \ + perturb the base simulation" + ); + } + } +} diff --git a/src/simlin-engine/src/db/ltm_module_tests.rs b/src/simlin-engine/src/db/ltm_module_tests.rs index 756d6e772..947dd36f2 100644 --- a/src/simlin-engine/src/db/ltm_module_tests.rs +++ b/src/simlin-engine/src/db/ltm_module_tests.rs @@ -438,7 +438,7 @@ fn test_discovery_dynamic_module_link_score_uses_composite() { .find(|v| v.name.contains("level\u{2192}custom_smooth")) .expect("discovery mode must emit the level→custom_smooth link score"); let eqn_text = match &link.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("module link score should be scalar, got {other:?}"), }; assert!( @@ -538,7 +538,7 @@ fn test_passthrough_module_link_score_uses_composite_on_real_output_port() { .find(|v| v.name == base_name) .expect("must emit the base level→custom_pt link score"); let eqn_text = match &link.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("module link score should be scalar, got {other:?}"), }; assert!( @@ -642,7 +642,7 @@ fn test_pathless_module_link_score_uses_unit_transfer() { continue; }; let eqn_text = match &link.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("module link score should be scalar, got {other:?}"), }; assert!( @@ -760,7 +760,7 @@ fn test_module_to_module_link_score_uses_target_composite() { .find(|v| v.name.contains("mod_a\u{2192}mod_b")) .expect("must emit the mod_a→mod_b link score"); let eqn_text = match &link.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("module link score should be scalar, got {other:?}"), }; assert_eq!( @@ -1542,7 +1542,7 @@ fn test_multi_output_loop_link_uses_per_exit_port_alias() { ) }); let alias_eqn = match &alias.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("alias should be scalar, got {other:?}"), }; // The alias must reference one of m's pathway vars (the pos-terminal @@ -1567,7 +1567,7 @@ fn test_multi_output_loop_link_uses_per_exit_port_alias() { .find(|v| v.name.contains("\u{205A}loop_score\u{205A}")) .expect("must emit a loop_score var"); let loop_eqn = match &loop_score.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("loop score should be scalar, got {other:?}"), }; assert!( @@ -1827,7 +1827,7 @@ fn test_multi_output_module_link_score_holds_document_order_first_live() { ) }); match &link.equation { - datamodel::Equation::Scalar(text) => text.clone(), + crate::db::LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("module link score should be scalar, got {other:?}"), } }; diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index fb5bbec4c..106fe1e40 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -5,7 +5,7 @@ use super::{compile_ltm_equation_fragment, scalarize_ltm_equation}; use crate::datamodel; use crate::db::{ - LtmLinkId, RefShape, ShapedLinkScore, SimlinDb, compute_layout, link_score_equation_text, + LtmLinkId, RefShape, ShapedLinkScore, SimlinDb, compute_layout, link_score_equation_text_shaped, sync_from_datamodel, }; use crate::test_common::TestProject; @@ -86,7 +86,7 @@ fn test_ltm_previous_module_var_uses_helper_rewrite() { let fragment = compile_ltm_equation_fragment( &db, "$⁚ltm⁚test_prev_module", - &datamodel::Equation::Scalar("PREVIOUS(producer)".to_string()), + &crate::db::LtmEquation::scalar("PREVIOUS(producer)".to_string()), source_model, sync.project, ) @@ -134,7 +134,10 @@ fn test_a2a_ltm_equation_fragment_compiles() { let fragment = compile_ltm_equation_fragment( &db, "$\u{205A}ltm\u{205A}test_a2a_link_score", - &datamodel::Equation::ApplyToAll(dims.clone(), "PREVIOUS(population) * 0.5".to_string()), + &crate::db::LtmEquation::apply_to_all( + dims.clone(), + "PREVIOUS(population) * 0.5".to_string(), + ), source_model, sync.project, ) @@ -260,7 +263,10 @@ fn test_a2a_ltm_previous_per_element() { let fragment = compile_ltm_equation_fragment( &db, "$\u{205A}ltm\u{205A}test_prev_per_elem", - &datamodel::Equation::ApplyToAll(dims.clone(), "PREVIOUS(population) * 0.5".to_string()), + &crate::db::LtmEquation::apply_to_all( + dims.clone(), + "PREVIOUS(population) * 0.5".to_string(), + ), source_model, sync.project, ) @@ -344,13 +350,16 @@ fn test_stock_to_flow_link_score_handles_apply_to_all() { let sync = sync_from_datamodel(&db, &project); let source_model = sync.models["main"].source; - // The stock-to-flow direction: population -> births + // The stock-to-flow direction: population -> births. The scalar Bare score + // (what assembly's sub-case (a) compiles) comes from the shape-aware query; + // stock-to-flow ignores `RefShape`, so `Bare` yields the same generator + // output as the (deleted) legacy `(from, to)`-keyed query did. let link_id = LtmLinkId::new(&db, "population".to_string(), "births".to_string()); - let lsv = link_score_equation_text(&db, link_id, source_model, sync.project); - - let lsv = lsv - .as_ref() - .expect("stock-to-flow link score should be generated for arrayed model"); + let ShapedLinkScore::Scored(lsv) = + link_score_equation_text_shaped(&db, link_id, RefShape::Bare, source_model, sync.project) + else { + panic!("stock-to-flow link score should be generated for arrayed model"); + }; // Before the fix, the equation would contain only "0" terms because // the flow_equation was "0" (ApplyToAll fell through the Scalar-only @@ -465,8 +474,8 @@ fn test_stock_to_flow_link_score_handles_arrayed() { // Each `births[e]` references `population[e]` -- a FixedIndex(e) ref -- // so the per-shape emission yields `population[e] -> births` link - // scores. The non-shaped `link_score_equation_text` would `scalarize` - // the result; use the shaped entry point so the arrayed equation + // scores. The scalar Bare score would `scalarize` the result; use the + // shaped entry point with the FixedIndex shape so the arrayed equation // survives intact. let link_id = LtmLinkId::new(&db, "population".to_string(), "births".to_string()); let result = link_score_equation_text_shaped( @@ -481,16 +490,17 @@ fn test_stock_to_flow_link_score_handles_arrayed() { }; let elements = match &lsv.equation { - datamodel::Equation::Arrayed(_, elements, _, _) => elements, + crate::db::LtmEquation::Arrayed { elements, .. } => elements, other => { - panic!("stock-to-arrayed-flow link score must be Equation::Arrayed, got: {other:?}") + panic!("stock-to-arrayed-flow link score must be LtmEquation::Arrayed, got: {other:?}") } }; assert!( !elements.is_empty(), "arrayed link score should have per-element slots" ); - for (elem, slot_eqn, _, _) in elements { + for (elem, arm) in elements { + let slot_eqn = &arm.text; // The flow's actual equation contents (`population`) must show up // in every slot -- before the fix this was a constant `(0)`. assert!( @@ -508,40 +518,48 @@ fn test_stock_to_flow_link_score_handles_arrayed() { #[test] fn test_scalarize_ltm_equation_arrayed_collapse() { - use datamodel::Equation::{ApplyToAll, Arrayed, Scalar}; + use crate::db::LtmEquation; + // Uses parseable equation bodies (`LtmEquation` parses each arm eagerly, so + // a placeholder like "first slot" would trip the augmentation-bug guard); + // scalarize only ever selects an arm, so the exact expression is irrelevant. + // // Arrayed with multiple per-element slots collapses to the *first* slot's text. - let multi = Arrayed( + let multi = LtmEquation::arrayed( vec!["region".to_string()], vec![ - ("nyc".to_string(), "first slot".to_string(), None, None), - ("boston".to_string(), "second slot".to_string(), None, None), + ("nyc".to_string(), "first_slot".to_string()), + ("boston".to_string(), "second_slot".to_string()), ], None, false, ); - assert!(matches!(scalarize_ltm_equation(multi), Scalar(text) if text == "first slot")); + assert!( + matches!(scalarize_ltm_equation(multi), LtmEquation::Scalar(arm) if arm.text == "first_slot") + ); // Arrayed with no slots but a Some(default) falls back to the default text. - let default_only = Arrayed( + let default_only = LtmEquation::arrayed( vec!["region".to_string()], vec![], - Some("default eqn".to_string()), + Some("default_eqn".to_string()), false, ); - assert!(matches!(scalarize_ltm_equation(default_only), Scalar(text) if text == "default eqn")); + assert!( + matches!(scalarize_ltm_equation(default_only), LtmEquation::Scalar(arm) if arm.text == "default_eqn") + ); // Arrayed with neither slots nor a default falls back to "0". - let empty = Arrayed(vec!["region".to_string()], vec![], None, false); - assert!(matches!(scalarize_ltm_equation(empty), Scalar(text) if text == "0")); + let empty = LtmEquation::arrayed(vec!["region".to_string()], vec![], None, false); + assert!(matches!(scalarize_ltm_equation(empty), LtmEquation::Scalar(arm) if arm.text == "0")); // ApplyToAll and Scalar inputs are preserved (text kept, dims dropped). assert!(matches!( - scalarize_ltm_equation(ApplyToAll(vec!["region".to_string()], "a2a eqn".to_string())), - Scalar(text) if text == "a2a eqn" + scalarize_ltm_equation(LtmEquation::apply_to_all(vec!["region".to_string()], "a2a_eqn".to_string())), + LtmEquation::Scalar(arm) if arm.text == "a2a_eqn" )); assert!( - matches!(scalarize_ltm_equation(Scalar("scalar eqn".to_string())), Scalar(text) if text == "scalar eqn") + matches!(scalarize_ltm_equation(LtmEquation::scalar("scalar_eqn".to_string())), LtmEquation::Scalar(arm) if arm.text == "scalar_eqn") ); } @@ -749,48 +767,12 @@ fn collect_agg_petals_groups_single_agg_circuits() { } } -/// Finding-1 regression: the `(from, to)`-keyed legacy `link_score_equation_text` -/// and the shape-aware `link_score_equation_text_shaped(.., Bare)` must derive -/// the SAME equation for a standard scalar Bare link score. The two twins had -/// silently drifted: the legacy query passed an EMPTY occurrence stream, so the -/// ceteris-paribus wrap's GH #517 reducer-freeze arm (`subtree_has_live_shape`) -/// froze a reducer whole, while the shaped query threaded the real stream and -/// recursed into it -- producing a different partial for `scale -> total` -/// whenever the live source `scale` appears bare inside a reducer of `total`. -/// The compiled fragment (assembly routes the standard scalar Bare score through -/// the legacy query) would then simulate an equation that disagrees with the one -/// `model_ltm_variables` reports/serializes. -#[test] -fn legacy_and_shaped_bare_score_agree_when_source_bare_in_reducer() { - let project = TestProject::new("bare_source_in_reducer") - .with_sim_time(0.0, 10.0, 1.0) - .named_dimension("D1", &["a", "b"]) - .array_aux("arr[D1]", "1") - .aux("scale", "pop * 0.01", None) - .aux("total", "scale + SUM(arr[*] * scale)", None) - .flow("growth", "total * 0.001", None) - .stock("pop", "1", &["growth"], &[], None) - .build_datamodel(); - - let db = SimlinDb::default(); - let sync = sync_from_datamodel(&db, &project); - let source_model = sync.models["main"].source; - - let link_id = LtmLinkId::new(&db, "scale".to_string(), "total".to_string()); - let legacy = link_score_equation_text(&db, link_id, source_model, sync.project) - .as_ref() - .expect("legacy scale -> total link score should exist"); - let shaped = - link_score_equation_text_shaped(&db, link_id, RefShape::Bare, source_model, sync.project); - let ShapedLinkScore::Scored(shaped) = shaped else { - panic!("shaped scale -> total Bare link score should be Scored"); - }; - - assert_eq!( - legacy.equation.source_text(), - shaped.equation.source_text(), - "the legacy (compiled/assembled) and shaped (emitted/reported) Bare link \ - scores must be byte-identical -- a divergence means the VM simulates a \ - different equation than is reported" - ); -} +// The Track A3 stage-2b regression `legacy_and_shaped_bare_score_agree_when_ +// source_bare_in_reducer` was removed here: it pinned that the deleted +// `(from, to)`-keyed `link_score_equation_text` and `link_score_equation_text_ +// shaped(.., Bare)` derived byte-identical equations. With the legacy query +// gone, assembly's sub-case (a) reads the shaped query directly, so the two +// derivations are one -- the divergence the test guarded against is now +// structurally impossible. The emitted (changed-LAST) text for that probe is +// still pinned by the `scalar_feeder_bare_in_hoisted_reducer` characterization +// golden (`db::ltm_char_tests`). diff --git a/src/simlin-engine/src/db/ltm_unified_tests.rs b/src/simlin-engine/src/db/ltm_unified_tests.rs index 83f18c248..19a6445a7 100644 --- a/src/simlin-engine/src/db/ltm_unified_tests.rs +++ b/src/simlin-engine/src/db/ltm_unified_tests.rs @@ -2984,7 +2984,7 @@ fn agg_aux_emitted_for_hoisted_reducer() { agg.dimensions ); assert!( - matches!(&agg.equation, crate::datamodel::Equation::Scalar(t) if t == "sum(pop[*])"), + matches!(&agg.equation, crate::db::LtmEquation::Scalar(arm) if arm.text == "sum(pop[*])"), "agg equation should be the reducer subexpr text; got: {:?}", agg.equation ); @@ -3319,7 +3319,7 @@ fn sliced_agg_link_scores_cover_only_the_read_rows() { .unwrap_or_else(|| panic!("expected synthetic agg {agg}; got: {names:?}")); assert!(agg_var.dimensions.is_empty()); assert!( - matches!(&agg_var.equation, crate::datamodel::Equation::Scalar(t) if t == "sum(pop[nyc, *])"), + matches!(&agg_var.equation, crate::db::LtmEquation::Scalar(arm) if arm.text == "sum(pop[nyc, *])"), "agg equation should be the sliced reducer text; got: {:?}", agg_var.equation ); diff --git a/src/simlin-engine/src/db/tests.rs b/src/simlin-engine/src/db/tests.rs index 2e48fe459..90f92846c 100644 --- a/src/simlin-engine/src/db/tests.rs +++ b/src/simlin-engine/src/db/tests.rs @@ -2093,8 +2093,9 @@ fn two_loop_project() -> datamodel::Project { /// recompilation of loop B's link-score bytecode FRAGMENT. /// /// The meaningful cache is the compiled fragment (`compile_ltm_var_fragment`), -/// not the equation-text query. Since the finding-1 fix, the equation-text -/// query `link_score_equation_text` reads the whole-model occurrence IR +/// not the equation-text query. The per-shape equation-text query +/// `link_score_equation_text_shaped(.., Bare)` (which `compile_ltm_var_fragment` +/// now sources from) reads the whole-model occurrence IR /// (`model_ltm_reference_sites`) so the compiled fragment matches the emitted /// one -- so editing ANY variable re-runs it. But it produces an UNCHANGED value /// for an unaffected edge, so salsa backdates it and the expensive diff --git a/src/simlin-engine/src/ltm_agg.rs b/src/simlin-engine/src/ltm_agg.rs index 05103eb2b..016522763 100644 --- a/src/simlin-engine/src/ltm_agg.rs +++ b/src/simlin-engine/src/ltm_agg.rs @@ -2410,7 +2410,7 @@ fn first_active_bare_arrayed_reducer( /// [`PerElementReducerRead`]: UnhoistedSourceRead::PerElementReducerRead /// /// Salsa-tracked, keyed on the interned [`LtmLinkId`] (the -/// `link_score_equation_text` idiom): the body's `reconstruct_model_variables` +/// per-link `compile_ltm_var_fragment` idiom): the body's `reconstruct_model_variables` /// is the codebase's one UN-tracked whole-model reconstruction (O(all model /// vars)), and this is its first per-edge caller -- tracking bounds that cost /// to once per `(edge, revision)` so the pinned-loop pass's and discovery diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index a1e1b44f8..35dff3dd9 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -19,6 +19,7 @@ use crate::ltm::{Loop, normalize_module_ref, split_node_subscript, strip_subscri use crate::variable::{Variable, identifier_set}; use std::collections::{HashMap, HashSet}; +use crate::db::LtmEquation; use crate::db::RefShape; use crate::db::ltm_ir::{ OccurrenceAxis, OccurrenceRef, OccurrenceSite, OtherDepVerdict, derive_other_dep_verdict, @@ -3056,7 +3057,7 @@ pub(crate) fn link_score_var_name(from: &str, to: &str, shape: &RefShape) -> Str /// Generate absolute loop score variables for all loops. /// /// Emits one `$⁚ltm⁚loop_score⁚{id}` entry per loop, returning the variable -/// name plus the *dimension-shaped* `datamodel::Equation` it should carry: +/// name plus the *dimension-shaped* typed [`LtmEquation`] it should carry: /// /// - **Scalar loops** (`dimensions` empty): `Equation::Scalar`, the product of /// the loop's link-score references. @@ -3091,7 +3092,7 @@ pub(crate) fn generate_loop_score_variables( emitted_link_score_names: &HashSet, dm_dims: &[datamodel::Dimension], overrides: &LoopLinkOverrides, -) -> Vec<(String, datamodel::Equation)> { +) -> Vec<(String, LtmEquation)> { let mut loop_vars = Vec::with_capacity(loops.len()); // Loop-score tracing is a benchmarking/diagnostic aid compiled in only @@ -3131,8 +3132,6 @@ pub(crate) fn generate_loop_score_variables( /// at power-of-two sample points plus every 10,000 loops. Enable it with /// `cargo run --release --example ltm_full_bench --features ltm_bench -- `. mod loop_score_trace { - use crate::datamodel; - #[cfg(feature = "ltm_bench")] pub(super) struct LoopScoreTrace { loop_score_bytes: u64, @@ -3153,7 +3152,7 @@ mod loop_score_trace { /// Accumulate `equation`'s text bytes and, on a sample point, log the /// running total alongside RSS. `n` is the 1-based loop index. - pub(super) fn record(&mut self, n: usize, equation: &datamodel::Equation) { + pub(super) fn record(&mut self, n: usize, equation: &crate::db::LtmEquation) { self.loop_score_bytes += equation_text_len(equation) as u64; if should_trace(n) { eprintln!( @@ -3176,16 +3175,20 @@ mod loop_score_trace { } } - /// Total equation-text bytes of a `datamodel::Equation`. + /// Total equation-text bytes of an `LtmEquation` (the diagnostic text). #[cfg(feature = "ltm_bench")] - fn equation_text_len(equation: &datamodel::Equation) -> usize { + fn equation_text_len(equation: &crate::db::LtmEquation) -> usize { + use crate::db::LtmEquation; match equation { - datamodel::Equation::Scalar(text) | datamodel::Equation::ApplyToAll(_, text) => { - text.len() - } - datamodel::Equation::Arrayed(_, elements, default, _) => { - elements.iter().map(|(_, eq, _, _)| eq.len()).sum::() - + default.as_ref().map(String::len).unwrap_or(0) + LtmEquation::Scalar(arm) | LtmEquation::ApplyToAll(_, arm) => arm.text.len(), + LtmEquation::Arrayed { + elements, default, .. + } => { + elements + .iter() + .map(|(_, arm)| arm.text.len()) + .sum::() + + default.as_ref().map(|a| a.text.len()).unwrap_or(0) } } } @@ -3244,7 +3247,7 @@ mod loop_score_trace { LoopScoreTrace } #[inline(always)] - pub(super) fn record(&mut self, _n: usize, _equation: &datamodel::Equation) {} + pub(super) fn record(&mut self, _n: usize, _equation: &crate::db::LtmEquation) {} #[inline(always)] pub(super) fn done(&self, _loop_count: usize) {} } @@ -3271,17 +3274,17 @@ fn all_links_resolve_bare(loop_item: &Loop, emitted: &HashSet) -> bool { }) } -/// Build the dimension-shaped `datamodel::Equation` for one loop's score +/// Build the dimension-shaped typed [`LtmEquation`] for one loop's score /// variable. See [`generate_loop_score_variables`] for the three cases. fn generate_dimensioned_loop_score_equation( loop_item: &Loop, emitted: &HashSet, dm_dims: &[datamodel::Dimension], overrides: &LoopLinkOverrides, -) -> Option { +) -> Option { if loop_item.dimensions.is_empty() { return try_generate_loop_score_equation(loop_item, emitted, overrides) - .map(datamodel::Equation::Scalar); + .map(LtmEquation::scalar); } // Prefer the compact ApplyToAll form whenever it is correct (every link // resolves through a Bare A2A name), regardless of whether per-slot @@ -3293,7 +3296,7 @@ fn generate_dimensioned_loop_score_equation( // compilation can synthesize a missing-dependency zero stub. if loop_item.slot_links.is_empty() || all_links_resolve_bare(loop_item, emitted) { return try_generate_loop_score_equation(loop_item, emitted, overrides) - .map(|text| datamodel::Equation::ApplyToAll(loop_item.dimensions.clone(), text)); + .map(|text| LtmEquation::apply_to_all(loop_item.dimensions.clone(), text)); } // Per-slot equations: enumerate the loop's full dimension element space @@ -3319,15 +3322,15 @@ fn generate_dimensioned_loop_score_equation( .iter() .map(|(t, l)| (t.as_str(), l.as_slice())) .collect(); - let mut elements = Vec::with_capacity(slot_keys.len()); + let mut elements: Vec<(String, String)> = Vec::with_capacity(slot_keys.len()); for tuple in &slot_keys { let text = match by_tuple.get(tuple.as_str()) { Some(links) => generate_link_product(links, emitted, None)?, None => "0".to_string(), }; - elements.push((tuple.clone(), text, None, None)); + elements.push((tuple.clone(), text)); } - Some(datamodel::Equation::Arrayed( + Some(LtmEquation::arrayed( loop_item.dimensions.clone(), elements, None, @@ -3366,9 +3369,9 @@ fn target_iterated_dim_names_canonical(to_var: &Variable) -> Vec { /// Exposed as `generate_link_score_equation_for_link` for use by tracked /// functions in `db.rs`. /// -/// Returns a [`datamodel::Equation`] whose variant matches the *target* -/// variable's shape: `Equation::Scalar` for a scalar target, -/// `Equation::ApplyToAll(target_dims, _)` for an arrayed target (so the +/// Returns a typed [`LtmEquation`] whose variant matches the *target* +/// variable's shape: scalar for a scalar target, +/// apply-to-all over `target_dims` for an arrayed target (so the /// compiler expands the formula per element). `target_dims` uses the /// target's datamodel dimension names; the link emission loop overwrites /// them with the link-score-dimensions policy result, which is the same @@ -3404,7 +3407,7 @@ pub(crate) fn generate_link_score_equation_for_link( dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, to_occurrences: &[OccurrenceSite], -) -> Result { +) -> Result { generate_link_score_equation( from, to, @@ -3436,7 +3439,7 @@ fn generate_link_score_equation( dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, to_occurrences: &[OccurrenceSite], -) -> Result { +) -> Result { // Check if this is a stock-to-flow link let is_stock_to_flow = matches!(all_vars.get(from), Some(Variable::Stock { .. })) && matches!(to_var, Variable::Var { is_flow: true, .. }); @@ -3557,10 +3560,10 @@ fn target_equation_dims(var: &Variable) -> Option> { /// Build the link-score [`Equation`] for a target with the given guard-form /// equation `text`: `Equation::Scalar` for a scalar target, /// `Equation::ApplyToAll(target_dims, text)` for an arrayed target. -fn link_score_equation_for_target(text: String, to_var: &Variable) -> Equation { +fn link_score_equation_for_target(text: String, to_var: &Variable) -> LtmEquation { match target_equation_dims(to_var) { - Some(dims) => Equation::ApplyToAll(dims, text), - None => Equation::Scalar(text), + Some(dims) => LtmEquation::apply_to_all(dims, text), + None => LtmEquation::scalar(text), } } @@ -3626,7 +3629,7 @@ fn build_arrayed_link_score_equation( dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, to_occurrences: &[OccurrenceSite], -) -> Result { +) -> Result { // The #511 iterated-dimension context for the per-slot partials: each // per-element slot can itself reference `from` by an iterated dimension // of the *target*'s dimension space. `target_ast_dims`' canonical names @@ -3707,12 +3710,7 @@ fn build_arrayed_link_score_equation( per_elem.iter().collect(); sorted_slots.sort_by(|a, b| a.0.cmp(b.0)); - let elements: Vec<( - String, - String, - Option, - Option, - )> = sorted_slots + let elements: Vec<(String, String)> = sorted_slots .iter() .enumerate() .map(|(slot, (elem, expr))| { @@ -3720,8 +3718,6 @@ fn build_arrayed_link_score_equation( Ok(( elem.as_str().to_string(), slot_equation(expr, gf_table_ref.as_deref(), slot as u16)?, - None, - None, )) }) .collect::>()?; @@ -3737,7 +3733,7 @@ fn build_arrayed_link_score_equation( }) .transpose()?; - Ok(Equation::Arrayed( + Ok(LtmEquation::arrayed( target_dim_names, elements, default_slot, @@ -3854,7 +3850,7 @@ fn generate_auxiliary_to_auxiliary_equation( dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, to_occurrences: &[OccurrenceSite], -) -> Result { +) -> Result { use crate::ast::Ast; let to_q = quote_ident(to.as_str()); @@ -4090,7 +4086,7 @@ fn generate_flow_to_stock_equation( stock: &str, flow_var: &Variable, stock_var: &Variable, -) -> Equation { +) -> LtmEquation { // Check if this flow is an inflow or outflow let is_inflow = if let Variable::Stock { inflows, .. } = stock_var { inflows.iter().any(|f| f.as_str() == flow) @@ -4161,7 +4157,7 @@ fn generate_stock_to_flow_equation( dim_ctx: Option<&crate::dimensions::DimensionsContext>, dep_dims: Option<&HashMap>>, to_occurrences: &[OccurrenceSite], -) -> Result { +) -> Result { // For stock-to-flow, we need to calculate how the stock influences the flow // This is similar to auxiliary-to-auxiliary but we know the 'from' is a stock use crate::ast::Ast; diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index 7487ba43a..013b17138 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -9,6 +9,7 @@ use super::*; use crate::common::{CanonicalDimensionName, CanonicalElementName}; +use crate::db::LtmEquation; use crate::dimensions::{Dimension, NamedDimension}; fn make_named_dimension(name: &str, elements: &[&str]) -> Dimension { @@ -3041,16 +3042,17 @@ fn arrayed_var_from_text( /// Look up the slot equation for `element` in an `Equation::Arrayed`, /// failing the test loudly if the equation isn't `Arrayed` or the slot /// is missing. -fn arrayed_slot<'a>(equation: &'a Equation, element: &str) -> &'a str { +fn arrayed_slot<'a>(equation: &'a crate::db::LtmEquation, element: &str) -> &'a str { + use crate::db::LtmEquation; match equation { - Equation::Arrayed(_, elements, _, _) => elements + LtmEquation::Arrayed { elements, .. } => elements .iter() - .find(|(e, _, _, _)| e == element) - .map(|(_, eqn, _, _)| eqn.as_str()) + .find(|(e, _)| e == element) + .map(|(_, arm)| arm.text.as_str()) .unwrap_or_else(|| { panic!("no slot for element {element:?} in arrayed equation: {equation:?}") }), - other => panic!("expected Equation::Arrayed, got: {other:?}"), + other => panic!("expected LtmEquation::Arrayed, got: {other:?}"), } } @@ -3160,7 +3162,7 @@ fn generate_link_score_equation_for_link_normal_target_is_ok() { ) .expect("a normal scalar target must produce a valid link-score equation"); let text = match &equation { - Equation::Scalar(t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("expected a scalar link score, got {other:?}"), }; // The non-source dep is frozen; the source stays live -- the partial @@ -3318,7 +3320,7 @@ fn link_score_for_with_lookup_scalar_target_wraps_partial_in_lookup() { ) .expect("a with-lookup scalar target must produce a valid link-score equation"); let text = match &equation { - Equation::Scalar(t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), other => panic!("expected a scalar link score, got {other:?}"), }; assert!( @@ -3360,9 +3362,9 @@ fn link_score_for_with_lookup_a2a_target_pins_shared_table() { ) .expect("a with-lookup A2A target must produce a valid link-score equation"); let text = match &equation { - Equation::ApplyToAll(dim_names, t) => { + LtmEquation::ApplyToAll(dim_names, arm) => { assert_eq!(dim_names, &["Region".to_string()]); - t.clone() + arm.text.clone() } other => panic!("expected an apply-to-all link score, got {other:?}"), }; @@ -3463,11 +3465,15 @@ fn test_arrayed_link_score_population_to_migration_pressure_fixed_nyc() { .unwrap(); match &equation { - Equation::Arrayed(eq_dims, _, default, _) => { + LtmEquation::Arrayed { + dims: eq_dims, + default, + .. + } => { assert_eq!(eq_dims, &["Region".to_string()]); assert!(default.is_none(), "no EXCEPT default expected"); } - other => panic!("expected Equation::Arrayed, got: {other:?}"), + other => panic!("expected LtmEquation::Arrayed, got: {other:?}"), } let nyc_slot = arrayed_slot(&equation, "nyc"); @@ -3659,7 +3665,7 @@ fn test_scalar_and_a2a_link_scores_keep_their_shapes() { ) .unwrap(); assert!( - matches!(equation, Equation::Scalar(_)), + matches!(equation, LtmEquation::Scalar(_)), "scalar target must yield Equation::Scalar; got: {equation:?}" ); @@ -3706,7 +3712,7 @@ fn test_scalar_and_a2a_link_scores_keep_their_shapes() { ) .unwrap(); match equation { - Equation::ApplyToAll(d, _) => assert_eq!(d, vec!["Region".to_string()]), + LtmEquation::ApplyToAll(d, _) => assert_eq!(d, vec!["Region".to_string()]), other => panic!("ApplyToAll target must yield Equation::ApplyToAll; got: {other:?}"), } } @@ -3776,11 +3782,11 @@ fn test_flow_to_stock_arrayed_subscripts_references() { let equation = generate_flow_to_stock_equation("growth", "pop", &flow, &stock); let text = match &equation { - Equation::ApplyToAll(dims, text) => { + LtmEquation::ApplyToAll(dims, arm) => { assert_eq!(dims, &vec!["region".to_string()]); - text + &arm.text } - other => panic!("arrayed stock must yield Equation::ApplyToAll; got: {other:?}"), + other => panic!("arrayed stock must yield LtmEquation::ApplyToAll; got: {other:?}"), }; // Every stock/flow occurrence carries the dimension subscript -- @@ -3817,7 +3823,7 @@ fn test_flow_to_stock_scalar_stays_bare() { let equation = generate_flow_to_stock_equation("births", "s", &flow, &stock); let text = match &equation { - Equation::Scalar(text) => text, + LtmEquation::Scalar(arm) => &arm.text, other => panic!("scalar stock must yield Equation::Scalar; got: {other:?}"), }; @@ -3853,7 +3859,7 @@ fn test_flow_to_stock_arrayed_outflow_sign() { let equation = generate_flow_to_stock_equation("deaths", "pop", &flow, &stock); let text = match &equation { - Equation::ApplyToAll(_, text) => text, + LtmEquation::ApplyToAll(_, arm) => &arm.text, other => panic!("arrayed stock must yield Equation::ApplyToAll; got: {other:?}"), }; @@ -3926,9 +3932,9 @@ fn loop_score_variables_scalar_loop_yields_scalar_equation() { let (name, equation) = &vars[0]; assert_eq!(name, "$\u{205A}ltm\u{205A}loop_score\u{205A}r1"); match equation { - Equation::Scalar(text) => { + LtmEquation::Scalar(arm) => { assert_eq!( - text, + &arm.text, &format!( "\"{}\" * \"{}\"", ls_name("pop", "births"), @@ -3937,7 +3943,7 @@ fn loop_score_variables_scalar_loop_yields_scalar_equation() { "scalar loop score must be the plain product of Bare link-score refs" ); } - other => panic!("scalar loop must yield Equation::Scalar; got: {other:?}"), + other => panic!("scalar loop must yield LtmEquation::Scalar; got: {other:?}"), } } @@ -4005,10 +4011,10 @@ fn loop_score_variables_a2a_without_slot_links_yields_apply_to_all() { assert_eq!(vars.len(), 1); let (_, equation) = &vars[0]; match equation { - Equation::ApplyToAll(eq_dims, text) => { + LtmEquation::ApplyToAll(eq_dims, arm) => { assert_eq!(eq_dims, &vec!["region".to_string()]); assert_eq!( - text, + &arm.text, &format!( "\"{}\" * \"{}\"", ls_name("pop", "births"), @@ -4017,7 +4023,7 @@ fn loop_score_variables_a2a_without_slot_links_yields_apply_to_all() { "Bare-A2A loop score must keep the compact ApplyToAll product form" ); } - other => panic!("Bare-A2A loop must yield Equation::ApplyToAll; got: {other:?}"), + other => panic!("Bare-A2A loop must yield LtmEquation::ApplyToAll; got: {other:?}"), } } @@ -4079,7 +4085,12 @@ fn loop_score_variables_slot_links_yield_arrayed_per_slot_equations() { let (name, equation) = &vars[0]; assert_eq!(name, "$\u{205A}ltm\u{205A}loop_score\u{205A}pin1"); match equation { - Equation::Arrayed(eq_dims, elements, default, _) => { + LtmEquation::Arrayed { + dims: eq_dims, + elements, + default, + .. + } => { assert_eq!(eq_dims, &vec!["scenario".to_string()]); assert!(default.is_none()); assert_eq!( @@ -4093,7 +4104,7 @@ fn loop_score_variables_slot_links_yield_arrayed_per_slot_equations() { // The det slot references det's FixedIndex link scores // subscripted at det (they are arrayed vars), and must not // reference low's. - let det_eq = &elements[0].1; + let det_eq = &elements[0].1.text; assert!( det_eq.contains(&format!("\"{}\"[det]", ls_name("heat[det]", "temp"))), "det slot must reference heat[det]→temp subscripted at [det]; got: {det_eq}" @@ -4102,13 +4113,13 @@ fn loop_score_variables_slot_links_yield_arrayed_per_slot_equations() { !det_eq.contains("low"), "det slot must not reference the low element's link scores; got: {det_eq}" ); - let low_eq = &elements[1].1; + let low_eq = &elements[1].1.text; assert!( low_eq.contains(&format!("\"{}\"[low]", ls_name("heat[low]", "temp"))), "low slot must reference heat[low]→temp subscripted at [low]; got: {low_eq}" ); } - other => panic!("slot_links loop must yield Equation::Arrayed; got: {other:?}"), + other => panic!("slot_links loop must yield LtmEquation::Arrayed; got: {other:?}"), } } @@ -4148,11 +4159,11 @@ fn loop_score_variables_missing_slot_scores_zero() { ); let (_, equation) = &vars[0]; match equation { - Equation::Arrayed(_, elements, _, _) => { + LtmEquation::Arrayed { elements, .. } => { assert_eq!(elements.len(), 3, "every declared element gets a slot"); let by_elem: std::collections::HashMap<&str, &str> = elements .iter() - .map(|(e, eq, _, _)| (e.as_str(), eq.as_str())) + .map(|(e, arm)| (e.as_str(), arm.text.as_str())) .collect(); assert!(by_elem["det"].contains("link_score")); assert_eq!( @@ -4161,7 +4172,7 @@ fn loop_score_variables_missing_slot_scores_zero() { ); assert_eq!(by_elem["high"], "0"); } - other => panic!("expected Equation::Arrayed; got: {other:?}"), + other => panic!("expected LtmEquation::Arrayed; got: {other:?}"), } } @@ -4213,24 +4224,24 @@ fn loop_score_variables_multi_dim_slot_tuples() { ); let (_, equation) = &vars[0]; match equation { - Equation::Arrayed(_, elements, _, _) => { + LtmEquation::Arrayed { elements, .. } => { // Row-major over declared order: nyc,young / nyc,old / // boston,young / boston,old. - let keys: Vec<&str> = elements.iter().map(|(e, _, _, _)| e.as_str()).collect(); + let keys: Vec<&str> = elements.iter().map(|(e, _)| e.as_str()).collect(); assert_eq!( keys, vec!["nyc,young", "nyc,old", "boston,young", "boston,old"] ); let by_elem: std::collections::HashMap<&str, &str> = elements .iter() - .map(|(e, eq, _, _)| (e.as_str(), eq.as_str())) + .map(|(e, arm)| (e.as_str(), arm.text.as_str())) .collect(); assert!(by_elem["nyc,young"].contains("link_score")); assert_eq!(by_elem["nyc,old"], "0"); assert_eq!(by_elem["boston,young"], "0"); assert!(by_elem["boston,old"].contains("link_score")); } - other => panic!("expected Equation::Arrayed; got: {other:?}"), + other => panic!("expected LtmEquation::Arrayed; got: {other:?}"), } } @@ -4281,10 +4292,10 @@ fn loop_score_variables_prefer_apply_to_all_when_all_links_bare() { ); let (_, equation) = &vars[0]; match equation { - Equation::ApplyToAll(eq_dims, text) => { + LtmEquation::ApplyToAll(eq_dims, arm) => { assert_eq!(eq_dims, &vec!["region".to_string()]); assert_eq!( - text, + &arm.text, &format!( "\"{}\" * \"{}\"", ls_name("pop", "births"), diff --git a/src/simlin-engine/src/ltm_finding_tests.rs b/src/simlin-engine/src/ltm_finding_tests.rs index 2acdbd4b7..64d6e7087 100644 --- a/src/simlin-engine/src/ltm_finding_tests.rs +++ b/src/simlin-engine/src/ltm_finding_tests.rs @@ -697,13 +697,13 @@ fn test_parse_link_offsets_a2a_expansion() { let ltm_vars = vec![ crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}birth_rate\u{2192}births".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["Region".to_string()], compile_directly: false, }, crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}scalar_a\u{2192}scalar_b".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec![], compile_directly: false, }, @@ -846,7 +846,7 @@ fn test_parse_link_offsets_scalar_source_projects_to_bare() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}scale\u{2192}growth".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["D1".to_string()], compile_directly: false, }]; @@ -910,7 +910,7 @@ fn test_parse_link_offsets_lower_dim_source_projects_and_broadcasts() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}boost\u{2192}growth".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["Region".to_string(), "Age".to_string()], compile_directly: false, }]; @@ -986,7 +986,7 @@ fn test_parse_link_offsets_fixed_index_from_a2a_expansion() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}pop[nyc]\u{2192}rel_pop".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["Region".to_string()], compile_directly: false, }]; @@ -1055,7 +1055,7 @@ fn test_parse_link_offsets_fixed_index_from_scalar() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}pop[nyc]\u{2192}total".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec![], compile_directly: false, }]; @@ -1187,13 +1187,13 @@ fn test_parse_link_offsets_dedupes_a2a_bare_over_fixed_index() { let ltm_vars = vec![ crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}pop\u{2192}share".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["Region".to_string()], compile_directly: false, }, crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}pop[nyc]\u{2192}share".to_string(), - equation: datamodel::Equation::Scalar(String::new()), + equation: crate::db::LtmEquation::scalar(String::new()), dimensions: vec!["Region".to_string()], compile_directly: false, }, diff --git a/src/simlin-engine/src/ltm_post.rs b/src/simlin-engine/src/ltm_post.rs index f406940ad..e34cf315a 100644 --- a/src/simlin-engine/src/ltm_post.rs +++ b/src/simlin-engine/src/ltm_post.rs @@ -2722,7 +2722,7 @@ mod tests { fn loop_element_index_scalar_loop() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec![], compile_directly: false, }]; @@ -2744,7 +2744,7 @@ mod tests { fn loop_element_index_named_1d() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string()], compile_directly: false, }]; @@ -2774,7 +2774,7 @@ mod tests { fn loop_element_index_mixed_2d() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string(), "Cohort".to_string()], compile_directly: false, }]; @@ -2810,7 +2810,7 @@ mod tests { fn resolve_1d_named() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string()], compile_directly: false, }]; @@ -2834,7 +2834,7 @@ mod tests { fn resolve_2d_named_indexed_row_major() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string(), "Cohort".to_string()], compile_directly: false, }]; @@ -2865,7 +2865,7 @@ mod tests { fn resolve_scalar_loop() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec![], compile_directly: false, }]; @@ -2889,7 +2889,7 @@ mod tests { fn resolve_dim_count_mismatch() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string()], compile_directly: false, }]; @@ -2922,7 +2922,7 @@ mod tests { fn resolve_unknown_element() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Region".to_string()], compile_directly: false, }]; @@ -2946,7 +2946,7 @@ mod tests { fn resolve_indexed_errors() { let ltm_vars = vec![crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec!["Cohort".to_string()], compile_directly: false, }]; @@ -2979,19 +2979,19 @@ mod tests { let ltm_vars = vec![ crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}link_score\u{205A}a\u{2192}b".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec![], compile_directly: false, }, crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}loop_score\u{205A}r1".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec![], compile_directly: false, }, crate::db::LtmSyntheticVar { name: "$\u{205A}ltm\u{205A}path\u{205A}foo\u{205A}0".to_string(), - equation: crate::datamodel::Equation::Scalar("1.0".to_string()), + equation: crate::db::LtmEquation::scalar("1.0".to_string()), dimensions: vec![], compile_directly: false, }, diff --git a/src/simlin-engine/src/variable.rs b/src/simlin-engine/src/variable.rs index fb0fd36aa..a46db1065 100644 --- a/src/simlin-engine/src/variable.rs +++ b/src/simlin-engine/src/variable.rs @@ -533,7 +533,7 @@ mod is_lookup_only_tests { } } -fn get_dimensions( +pub(crate) fn get_dimensions( dimensions: &[datamodel::Dimension], names: &[DimensionName], ) -> Result, EquationError> { diff --git a/src/simlin-engine/tests/integration/ltm_array_agg.rs b/src/simlin-engine/tests/integration/ltm_array_agg.rs index 7c065b5ec..278eb3617 100644 --- a/src/simlin-engine/tests/integration/ltm_array_agg.rs +++ b/src/simlin-engine/tests/integration/ltm_array_agg.rs @@ -64,8 +64,8 @@ use simlin_engine::datamodel::{self, Dimension}; use simlin_engine::db::{ - DetectedLoopPolarity, DiagnosticError, DiagnosticSeverity, LtmSyntheticVar, SimlinDb, - collect_all_diagnostics, compile_project_incremental, model_detected_loops, + DetectedLoopPolarity, DiagnosticError, DiagnosticSeverity, LtmEquation, LtmSyntheticVar, + SimlinDb, collect_all_diagnostics, compile_project_incremental, model_detected_loops, model_element_causal_edges, model_ltm_variables, reclassify_loops_from_results, set_project_ltm_discovery_mode, set_project_ltm_enabled, sync_from_datamodel_incremental, }; @@ -5339,9 +5339,9 @@ fn rank_frozen_subtree_link_score_scores_correctly() { // link-score names their loop-score equations reference. fn eqn_text(v: &LtmSyntheticVar) -> &str { match &v.equation { - datamodel::Equation::Scalar(t) => t, - datamodel::Equation::ApplyToAll(_, t) => t, - datamodel::Equation::Arrayed(..) => panic!("unexpected Arrayed loop score"), + LtmEquation::Scalar(arm) => &arm.text, + LtmEquation::ApplyToAll(_, arm) => &arm.text, + LtmEquation::Arrayed { .. } => panic!("unexpected Arrayed loop score"), } } let loop_scores: Vec<&LtmSyntheticVar> = ltm @@ -5378,10 +5378,10 @@ fn rank_frozen_subtree_link_score_scores_correctly() { .vars .iter() .find(|v| match &v.equation { - datamodel::Equation::ApplyToAll(dims, text) => { - dims.len() == 1 && dims[0] == "Region" && text == "rank(pop, 1)" + LtmEquation::ApplyToAll(dims, arm) => { + dims.len() == 1 && dims[0] == "Region" && arm.text == "rank(pop, 1)" } - datamodel::Equation::Scalar(_) | datamodel::Equation::Arrayed(..) => false, + LtmEquation::Scalar(_) | LtmEquation::Arrayed { .. } => false, }) .expect("RANK(pop, 1) must be emitted as an arrayed aggregate helper") .name @@ -5520,8 +5520,8 @@ fn gh525_two_reference_partially_iterated_row_sum_scores() { // per-(row, element) names, with real non-zero post-startup values. let eqn_text = |v: &LtmSyntheticVar| -> String { match &v.equation { - datamodel::Equation::Scalar(t) => t.clone(), - datamodel::Equation::ApplyToAll(_, t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), + LtmEquation::ApplyToAll(_, arm) => arm.text.clone(), other => format!("{other:?}"), } }; @@ -5587,12 +5587,16 @@ fn gh525_two_reference_partially_iterated_row_sum_scores() { .filter(|v| v.name.starts_with(LOOP_SCORE_PREFIX)) { let text = match &lv.equation { - datamodel::Equation::Scalar(t) => t.clone(), - datamodel::Equation::ApplyToAll(_, t) => t.clone(), - datamodel::Equation::Arrayed(_, slots, default, _) => { - let mut t: String = slots.iter().map(|(_, eq, _, _)| eq.clone()).collect(); + LtmEquation::Scalar(arm) => arm.text.clone(), + LtmEquation::ApplyToAll(_, arm) => arm.text.clone(), + LtmEquation::Arrayed { + elements: slots, + default, + .. + } => { + let mut t: String = slots.iter().map(|(_, arm)| arm.text.clone()).collect(); if let Some(d) = default { - t.push_str(d); + t.push_str(&d.text); } t } @@ -5856,7 +5860,7 @@ fn mixed_bare_and_per_element_edge_resolver_precedence() { ); let eqn_text = |v: &LtmSyntheticVar| -> String { match &v.equation { - datamodel::Equation::Scalar(t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), other => format!("{other:?}"), } }; @@ -5969,7 +5973,7 @@ fn per_element_hop_in_mixed_scalar_cycle_scores() { let mut saw_per_element_ref = 0usize; for lv in &loop_vars { let text = match &lv.equation { - datamodel::Equation::Scalar(t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), other => format!("{other:?}"), }; assert!( @@ -6372,7 +6376,7 @@ fn per_element_body_with_iterated_other_dep_scores() { // unresolvable bare dimension index. let var = ltm_var(<m.vars, &name); let eqn = match &var.equation { - datamodel::Equation::Scalar(t) => t.clone(), + LtmEquation::Scalar(arm) => arm.text.clone(), other => format!("{other:?}"), }; assert!( @@ -7817,8 +7821,8 @@ fn aligned_partial_reduce_emissions_stay_byte_identical() { ABS((inflow[d1\u{B7}a] - PREVIOUS(inflow[d1\u{B7}a]))), 0) * \ SIGN((matrix[d1\u{B7}a,d2\u{B7}x] - PREVIOUS(matrix[d1\u{B7}a,d2\u{B7}x])))"; match &emitted[0].equation { - datamodel::Equation::Scalar(text) => assert_eq!( - text, golden, + LtmEquation::Scalar(arm) => assert_eq!( + &arm.text, golden, "the aligned per-(row, slot) equation text must stay byte-identical" ), other => panic!("aligned per-(row, slot) score must be scalar; got {other:?}"), diff --git a/src/simlin-engine/tests/integration/simulate_ltm.rs b/src/simlin-engine/tests/integration/simulate_ltm.rs index 74e79d365..52f649ab3 100644 --- a/src/simlin-engine/tests/integration/simulate_ltm.rs +++ b/src/simlin-engine/tests/integration/simulate_ltm.rs @@ -17,7 +17,7 @@ use simlin_engine::common::{Canonical, Ident}; #[allow(deprecated)] use simlin_engine::db::model_element_loop_circuits; use simlin_engine::db::{ - DetectedLoop, DetectedLoopPolarity, SimlinDb, causal_graph_from_edges, + DetectedLoop, DetectedLoopPolarity, LtmEquation, SimlinDb, causal_graph_from_edges, causal_graph_from_element_edges, compile_project_incremental, model_causal_edges, model_cycle_partitions, model_detected_loops, model_element_causal_edges, model_element_cycle_partitions, model_loop_circuits, model_loop_circuits_tiered, @@ -6856,7 +6856,8 @@ fn test_iterated_dim_subscript_link_score_is_bare_and_simulates() { // The `level -> row_val` edge is same-dimension A2A (both over Region), // so the link score is `Equation::ApplyToAll` over Region (per-element). match &level_to_row_val.equation { - simlin_engine::datamodel::Equation::ApplyToAll(dims, text) => { + LtmEquation::ApplyToAll(dims, arm) => { + let text = &arm.text; assert_eq!( dims, &vec!["Region".to_string()], @@ -7311,7 +7312,7 @@ fn test_disjoint_dim_arrayed_target_per_source_element_link_scores() { ); // Each per-source-element link score is Equation::Arrayed over target's dims. match &m_var.equation { - simlin_engine::datamodel::Equation::Arrayed(dims, elements, _, _) => { + LtmEquation::Arrayed { dims, elements, .. } => { assert_eq!( dims, &vec!["D1".to_string(), "D2".to_string()], @@ -7324,8 +7325,8 @@ fn test_disjoint_dim_arrayed_target_per_source_element_link_scores() { let slot = |elem: &str| -> &str { elements .iter() - .find(|(e, _, _, _)| e == elem) - .map(|(_, eq, _, _)| eq.as_str()) + .find(|(e, _)| e == elem) + .map(|(_, arm)| arm.text.as_str()) .unwrap_or_else(|| panic!("slot {elem:?} not found in {elements:?}")) }; let ax = slot("a,x"); From 390939646e68b32980b1be279735bec3785d24a6 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 08:24:07 -0700 Subject: [PATCH 08/14] engine: reattach the misplaced LTM wrap rustdocs Comment-only. Inserting other_dep_verdict between wrap_non_matching_in_previous and its rustdoc left the wrap undocumented and gave the verdict helper a doc block describing live_source / other_deps / iter_ctx / out.live_ref / dims_ctx -- none of which are its parameters. The same accident already existed for classify_expr0_subscript_shape, whose doc sat on the bool-returning is_literal_element_index (so that predicate was documented as "Rules: any Wildcard -> RefShape::Wildcard"); this branch's edits appended new cfg(test) rationale into that wrong block, so it is corrected here rather than left to rot. Both blocks are also brought in step with the code they now describe: the wrap takes a &WrapCtx rather than positional parameters, its iterated-dim normalization is driven by the occurrence IR's shape (iter_ctx now supplies only the other-dep arity/mapping facts and the dimension-name index guard's fallback), and it threads a structural path; the shape classifier's rule list was missing the #511 Bare and #525 PerElement arms it actually implements. Two narrower staleness fixes ride along. A link_scores comment still pointed at `substituted`, the caller-side local this branch deleted when reducer substitution became a post-transform lowering. And OccurrenceSite's "every field has a named A2b/A3 consumer" claim was false once the wrap turned out to reach the already_lagged and in_reducer decisions structurally: target_element, routing, in_reducer, reducer_keys, and already_lagged have no production reader today. Saying so plainly -- and recording that each is pinned at the IR level, which is what keeps a walker regression catchable without one -- is more useful to the next reader than a claim they would disprove in a grep. --- src/simlin-engine/src/db/ltm/link_scores.rs | 5 +- src/simlin-engine/src/db/ltm_ir.rs | 17 ++- src/simlin-engine/src/ltm_augment.rs | 133 +++++++++++--------- 3 files changed, 89 insertions(+), 66 deletions(-) diff --git a/src/simlin-engine/src/db/ltm/link_scores.rs b/src/simlin-engine/src/db/ltm/link_scores.rs index b4e1b10e5..bb19b05e1 100644 --- a/src/simlin-engine/src/db/ltm/link_scores.rs +++ b/src/simlin-engine/src/db/ltm/link_scores.rs @@ -3493,8 +3493,9 @@ pub(super) fn emit_agg_to_target_link_scores( .collect(); let mut edge_vars: Vec = Vec::new(); for element in &cartesian_subscripts(&dim_element_lists) { - // The partial is built around the *bare* agg names (which is - // what `substituted` holds); `source_pins_for_target` then + // The partial is built around the *bare* agg names (what the + // generator's post-transform `reducer_subst` lowering yields); + // `source_pins_for_target` then // pins each arrayed agg -- the live one AND any frozen // co-agg -- to its projected `agg[]`, matching // `agg_name_for_target` for the live agg (the full element diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index dfa6a202b..05012dde8 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -1188,9 +1188,20 @@ pub(crate) enum OccurrenceRouting { /// One classified reference occurrence over a target equation, the finer /// substrate both LTM consumers project from (edge emission is a per-edge /// dedup of these; the transform selects a live SET by shape and names the -/// first non-index-nested occurrence as the normalizer). Every field maps to -/// a spec §3 requirement with a named A2b/A3 consumer -- there are no -/// speculative fields. +/// first non-index-nested occurrence as the normalizer). Every field maps to a +/// spec §3 requirement, but they are not all consumed by the same code today: +/// the ceteris-paribus wrap reads `site_id` (by path), `reference`, `shape`, +/// `axes`, and `index_nested`; `db::module_link_score_equation` reads +/// `reference` for the document-order module-output pick. `target_element`, +/// `routing`, `in_reducer`, `reducer_keys`, and `already_lagged` are classified +/// facts the wrap does not *need* to read -- it reaches the same decisions +/// structurally (it never descends into a `PREVIOUS`/`INIT` call, so +/// `already_lagged` content is untouched by construction; the reducer-freeze arm +/// asks `subtree_has_live_shape` rather than reading `in_reducer`). They are +/// carried because they are the same walk's output and the remaining GH #965 +/// generation-half work consumes them, and each is pinned at the IR level +/// (`db::ltm_ir_tests`) so a walker regression is caught even with no production +/// reader. Do not add a field with neither a production consumer nor an IR pin. #[derive(Debug, Clone, PartialEq, Eq, salsa::Update)] pub(crate) struct OccurrenceSite { /// Stable, deterministic occurrence identity within `to`'s equation. diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 35dff3dd9..1c5bfeb4d 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -452,31 +452,6 @@ fn other_dep_axis_lines_up( crate::ltm_agg::iterated_axis_slot_elements(d, dep_dim_name.as_ref(), &elems, dim_ctx).is_some() } -/// Classify an `Expr0` subscript's shape based on its indices. -/// -/// Mirrors `db::ltm_ir::resolve_literal_index`'s classification logic but at -/// the `Expr0` (parsed-AST) level. `#[cfg(test)]`: the ceteris-paribus wrap no -/// longer re-derives shape on `Expr0` (it reads the occurrence IR); this sibling -/// survives only for the wrap unit tests' occurrence builder and the alignment -/// gate -- see [`resolve_literal_element_index`]. Each input string in -/// `source_dim_elements` is the canonical lowercase element name for the -/// corresponding source dimension, in source-declared order. -/// -/// Rules: -/// - any `IndexExpr0::Wildcard` → `RefShape::Wildcard` -/// - all indices are literal element names that match the source's -/// declared elements (or parseable integer literals for indexed -/// dimensions) → `RefShape::FixedIndex(canonical_names)` -/// - otherwise (StarRange, DimPosition, Range, non-literal Expr, or a -/// literal that doesn't match) → `RefShape::DynamicIndex` -/// -/// The match tries each index against the dimension at that position -/// first, then falls back to scanning all dimensions. This keeps the -/// classifier robust when callers pass dimensions in source-declared -/// order but the subscript indices may not align 1:1 with dimension -/// positions in unusual cases. Defensive `DynamicIndex` for unknown -/// names ensures the worst case is over-conservative wrapping rather -/// than incorrectly matching the live shape. /// Whether a single subscript index is a "literal element" reference -- /// i.e., a `Var` naming a known dimension element or an integer literal. /// These are dimension references at runtime, not variable references, @@ -558,6 +533,33 @@ fn resolve_literal_element_index( } } +/// Classify an `Expr0` subscript's shape based on its indices. +/// +/// Mirrors `db::ltm_ir::resolve_literal_index`'s classification logic but at +/// the `Expr0` (parsed-AST) level. `#[cfg(test)]`: the ceteris-paribus wrap no +/// longer re-derives shape on `Expr0` (it reads the occurrence IR); this sibling +/// survives only for the wrap unit tests' occurrence builder and the alignment +/// gate -- see [`resolve_literal_element_index`]. Each input string in +/// `source_dim_elements` is the canonical lowercase element name for the +/// corresponding source dimension, in source-declared order. +/// +/// Rules: +/// - any `IndexExpr0::Wildcard` → `RefShape::Wildcard` +/// - an iterated-dimension subscript on the live source (all axes iterated) +/// → `RefShape::Bare`; a mixed iterated+literal one → `RefShape::PerElement` +/// - all indices are literal element names that match the source's +/// declared elements (or parseable integer literals for indexed +/// dimensions) → `RefShape::FixedIndex(canonical_names)` +/// - otherwise (StarRange, DimPosition, Range, non-literal Expr, or a +/// literal that doesn't match) → `RefShape::DynamicIndex` +/// +/// The literal pass tries each index against the dimension at that position +/// first, then falls back to scanning all dimensions. This keeps the +/// classifier robust when callers pass dimensions in source-declared +/// order but the subscript indices may not align 1:1 with dimension +/// positions in unusual cases. Defensive `DynamicIndex` for unknown +/// names ensures the worst case is over-conservative wrapping rather +/// than incorrectly matching the live shape. #[cfg(test)] fn classify_expr0_subscript_shape( indices: &[IndexExpr0], @@ -764,6 +766,29 @@ fn child_path(path: &[u16], i: u16) -> Vec { v } +/// The `Collapse` / `Mismatch` / `NotIterated` verdict for an iterated-dimension +/// subscript on a NON-live-source dependency, derived from the occurrence IR's +/// per-axis classification (`node_occ.axes`) plus the two arity facts the +/// occurrence does not carry: the dep's declared arity (from `iter_ctx.dep_dims`) +/// and the target's iterated-dim count. Delegates to the single-sourced +/// [`derive_other_dep_verdict`] -- the SAME rule `db::ltm_ir` feeds the edge +/// emitter -- so the wrap and the emitter cannot disagree on the verdict. +/// +/// `None` (no recorded occurrence, or no `iter_ctx` -- the scalar/agg callers +/// with no iterated-dimension space) yields `NotIterated`: there is no iterated +/// collapse to perform. +fn other_dep_verdict( + node_occ: Option<&OccurrenceSite>, + dep: &str, + iter_ctx: Option<&IteratedDimCtx<'_>>, +) -> OtherDepVerdict { + let (Some(occ), Some(ic)) = (node_occ, iter_ctx) else { + return OtherDepVerdict::NotIterated; + }; + let dep_arity = ic.dep_dims.and_then(|m| m.get(dep)).map(|d| d.len()); + derive_other_dep_verdict(&occ.axes, dep_arity, ic.target_iterated_dims.len()) +} + /// Walk an `Expr0` tree and wrap variable references in `PREVIOUS()` except /// those whose access shape matches the live shape for the given source, /// recording into `out.live_ref` the *first* `live_source` occurrence left @@ -771,28 +796,37 @@ fn child_path(path: &[u16], i: u16) -> Vec { /// `out.other_dep_mismatch` whether a GH #526 mismatched other-dep /// subscript doomed the changed-first form. /// -/// `live_source` identifies the source variable whose live shape is held -/// out from `PREVIOUS` wrapping. `live_shape` declares which AST occurrences -/// of that source remain live; all other occurrences (and all references -/// to other sources in the same expression) are wrapped. +/// `ctx.live_source` identifies the source variable whose live shape is held +/// out from `PREVIOUS` wrapping. `ctx.live_shape` declares which AST +/// occurrences of that source remain live; all other occurrences (and all +/// references to other sources in the same expression) are wrapped. /// -/// `other_deps` is the set of canonical idents for non-`live_source` +/// `ctx.other_deps` is the set of canonical idents for non-`live_source` /// dependencies that must be wrapped; nodes referencing names not in this /// set and not equal to `live_source` are left alone (function names and /// unknown identifiers). Indices of subscripts are recursively transformed /// even when the outer subscript matches the live shape, so nested /// references like `arr[other_var]` still get wrapped. /// -/// `iter_ctx` carries the GH #511 iterated-dimension context (the live +/// `ctx.iter_ctx` carries the GH #511 iterated-dimension context (the live /// source's declared dimension names + the target equation's iterated -/// dimensions + a `DimensionsContext` for the mapped case); when `Some`, -/// an iterated-dimension subscript is normalized to a bare `Var` *before* -/// the live/PREVIOUS dispatch -- so `row_sum[D1]` (a same-element reference -/// over the target's own `D1`) becomes either the live ref (`live_shape == -/// Bare`) or `PREVIOUS(Var(row_sum))` (a `Var` arg, which codegen accepts, -/// vs the `PREVIOUS(Subscript(...))` the pre-#511 code produced). Pass +/// dimensions + a `DimensionsContext` for the mapped case). An +/// iterated-dimension subscript on the live source is normalized to a bare +/// `Var` *before* the live/PREVIOUS dispatch -- so `row_sum[D1]` (a +/// same-element reference over the target's own `D1`) becomes either the live +/// ref (`live_shape == Bare`) or `PREVIOUS(Var(row_sum))` (a `Var` arg, which +/// codegen accepts, vs the `PREVIOUS(Subscript(...))` the pre-#511 code +/// produced). The normalization itself is driven by the occurrence IR's shape +/// (`ctx.occ`), not by `iter_ctx`; `iter_ctx` remains the other-dep verdict's +/// arity/mapping source and the dimension-name index guard's fallback. Pass /// `None` for callers whose live source is scalar (no source subscripts). /// +/// `ctx.occ` is the occurrence IR for the slot being wrapped -- the single +/// classifier family. `path` is the structural child-index path of `expr` +/// within that slot (empty at the slot root), tracked as the recursion +/// descends so it equals the occurrence's `SiteId`; see [`OccurrenceLookup`] +/// and [`child_path`]. +/// /// `out.live_ref` ends up holding the bare `Var(live_source)` for a `Bare` /// shape, or the (already index-transformed) `Subscript(live_source, ...)` /// for `FixedIndex`/`Wildcard`/`DynamicIndex`. Callers use this captured @@ -803,35 +837,12 @@ fn child_path(path: &[u16], i: u16) -> Vec { /// multi-dimensional) bare `live_source`, which would be a dimension error /// in a scalar link-score equation. /// -/// `dims_ctx` is the project-wide dimensions context used by +/// `ctx.dims_ctx` is the project-wide dimensions context used by /// [`qualify_element_index`] to recognize (and qualify) subscript indices /// that name dimension elements -- so they are never PREVIOUS-wrapped as if /// they were causal references (GH #587). `None` (test-only callers, or /// paths without project dims in scope) disables qualification, keeping the /// conservative wrapping behavior. -/// The `Collapse` / `Mismatch` / `NotIterated` verdict for an iterated-dimension -/// subscript on a NON-live-source dependency, derived from the occurrence IR's -/// per-axis classification (`node_occ.axes`) plus the two arity facts the -/// occurrence does not carry: the dep's declared arity (from `iter_ctx.dep_dims`) -/// and the target's iterated-dim count. Delegates to the single-sourced -/// [`derive_other_dep_verdict`] -- the SAME rule `db::ltm_ir` feeds the edge -/// emitter -- so the wrap and the emitter cannot disagree on the verdict. -/// -/// `None` (no recorded occurrence, or no `iter_ctx` -- the scalar/agg callers -/// with no iterated-dimension space) yields `NotIterated`: there is no iterated -/// collapse to perform. -fn other_dep_verdict( - node_occ: Option<&OccurrenceSite>, - dep: &str, - iter_ctx: Option<&IteratedDimCtx<'_>>, -) -> OtherDepVerdict { - let (Some(occ), Some(ic)) = (node_occ, iter_ctx) else { - return OtherDepVerdict::NotIterated; - }; - let dep_arity = ic.dep_dims.and_then(|m| m.get(dep)).map(|d| d.len()); - derive_other_dep_verdict(&occ.axes, dep_arity, ic.target_iterated_dims.len()) -} - fn wrap_non_matching_in_previous( expr: Expr0, ctx: &WrapCtx<'_>, From 17d4e7c033e3de1f79147acc8edcf6355ec86c4e Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 08:54:50 -0700 Subject: [PATCH 09/14] engine: quote LTM idents the lexer cannot read bare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit quote_ident spelled an identifier bare whenever every character was alphanumeric or underscore. A leading digit satisfies that but cannot START an identifier: the equation lexer reads `1stock` as the number 1 followed by the identifier `stock`. `"1stock"` is a legal quoted XMILE name, so a valid model produced a link score whose partial carried print_eqn's quoted `"1stock"` beside a bare `1stock` in the guard form's SIGN factor -- one equation, two spellings of one name, and the whole thing unparseable. Before the typed-equation work that meant a silently-degraded score; after it, the eager arm parse also tripped the augmentation-bug debug_assert, so a debug build aborted inside a salsa query on ordinary user input. quote_ident now also consults ast::needs_quoting -- the leading-character rule print_ident already applies on the print_eqn path -- so the two spellings cannot disagree. It deliberately does NOT delegate outright: U+00B7 IS XID_Continue, so needs_quoting alone would spell a module-output composite bare (`mod·out1`). That parses, but it would rewrite the emitted text of every module-composite link score for no bug. Requiring BOTH predicates can only ever add quotes, which is why every characterization golden is byte-identical. The debug_assert in LtmArm::new is removed rather than kept alongside the root-cause fix. Its premise -- a generated equation is a print_eqn re-print, so a parse failure can only be an augmentation bug -- was false, and asserting on it converted the pre-existing warn-and-skip degradation into an abort on a valid model. The loud path is the diagnostic that was already there: no AST means no bytecode, so model_ltm_fragment_diagnostics warns that the variable holds a layout slot reading constant 0. Generator bugs still fail loudly in tests via the numeric pins and the goldens. The regression pin asserts the arm has a parsed AST, not merely that the text contains a quote, so it catches any unparseable generated equation rather than this one spelling; mutation-verified by restoring the old predicate. --- src/simlin-engine/src/ast/mod.rs | 14 +++++- src/simlin-engine/src/db/ltm/equation.rs | 34 +++++++------- src/simlin-engine/src/db/ltm_tests.rs | 57 ++++++++++++++++++++++++ src/simlin-engine/src/ltm_augment.rs | 26 +++++++++-- 4 files changed, 111 insertions(+), 20 deletions(-) diff --git a/src/simlin-engine/src/ast/mod.rs b/src/simlin-engine/src/ast/mod.rs index c668ce91f..de97fd9d9 100644 --- a/src/simlin-engine/src/ast/mod.rs +++ b/src/simlin-engine/src/ast/mod.rs @@ -457,8 +457,18 @@ fn binary_op_latex(op: BinaryOp) -> BinaryOpLatex { /// Check whether a canonicalized identifier needs double-quoting to be /// re-parseable. Names containing characters outside XID_Start/XID_Continue -/// (like `$`, `⁚`, `/`) must be quoted. -fn needs_quoting(canonical: &str) -> bool { +/// (like `$`, `⁚`, `/`) must be quoted -- as must a name whose FIRST character +/// is not `XID_Start` even if every character is alphanumeric (`1stock`, a legal +/// quoted XMILE name: bare, the lexer reads the number `1` then the identifier +/// `stock`). +/// +/// `pub(crate)` because this is the single "can this name be spelled bare" +/// predicate: `print_ident` uses it for the `print_eqn` path and +/// `ltm_augment::quote_ident` for LTM's generated guard forms. A second +/// implementation drifts -- `quote_ident` previously tested "alphanumeric or +/// `_`", which a leading digit satisfies, so an LTM equation mixed a +/// `print_eqn`-quoted `"1stock"` with a bare `1stock` and failed to parse. +pub(crate) fn needs_quoting(canonical: &str) -> bool { let mut chars = canonical.chars(); match chars.next() { None => return true, diff --git a/src/simlin-engine/src/db/ltm/equation.rs b/src/simlin-engine/src/db/ltm/equation.rs index 772da1344..149f1cbea 100644 --- a/src/simlin-engine/src/db/ltm/equation.rs +++ b/src/simlin-engine/src/db/ltm/equation.rs @@ -50,21 +50,25 @@ impl LtmArm { /// Parse `text` into the authoritative AST once, at the generation /// (source-format) boundary. pub fn new(text: String) -> Self { - let expr = match Expr0::new(&text, LexerType::Equation) { - Ok(expr) => expr, - Err(_) => { - // A generated LTM equation is always a `print_eqn` re-print or - // a guard-form assembly of already-parsed sub-expressions, so a - // parse failure here means a bug in the augmentation layer, not - // bad user input. Degrade exactly as the old text path did (an - // unparseable equation carried no AST and compiled to no - // bytecode -> the fragment is dropped and - // `model_ltm_fragment_diagnostics` warns) rather than panicking - // -- libsimlin release builds are panic=abort. - debug_assert!(false, "LTM generated equation failed to parse: {text}"); - None - } - }; + // Degrade exactly as the old text path did: an unparseable equation + // carries no AST and compiles to no bytecode, so `compile_phase` sees an + // empty expr list, `flow_bytecodes` stays `None`, and + // `model_ltm_fragment_diagnostics` warns that the variable keeps a + // layout slot with no bytecode. That warning IS the loud path -- there + // is deliberately no assertion here. + // + // An earlier revision did `debug_assert!(false, ..)`, on the theory that + // a generated equation is always a `print_eqn` re-print and so a parse + // failure could only be an augmentation-layer bug. That theory was + // wrong, and the assert made a VALID model abort a debug build: a + // canonical name the lexer cannot read bare (`1stock`) was emitted + // unquoted by `ltm_augment::quote_ident`. The root cause is fixed there + // (it now shares `ast::needs_quoting` with `print_ident`), but the + // degradation stays non-fatal on principle -- this runs inside a salsa + // query on the ordinary read path, where aborting on user input is + // strictly worse than the warning, and libsimlin release builds are + // panic=abort. + let expr = Expr0::new(&text, LexerType::Equation).unwrap_or_default(); Self { text, expr } } } diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index 106fe1e40..be8e76fb4 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -563,6 +563,63 @@ fn test_scalarize_ltm_equation_arrayed_collapse() { ); } +/// A canonical name that cannot be spelled as a BARE identifier -- one whose +/// first character is not `XID_Start` -- must be quoted in every generated LTM +/// equation, so the equation parses. +/// +/// XMILE lets a modeler quote any name, so `"1stock"` is a legal variable and +/// canonicalizes to `1stock`. The equation lexer, though, only starts an +/// identifier on `XID_Start`/`_`: bare `1stock` lexes as the number `1` +/// followed by the identifier `stock`, which is a parse error. `quote_ident` +/// used to test "every char is alphanumeric or `_`", which a leading digit +/// satisfies, so the guard form was emitted with a bare `1stock` and the whole +/// link score failed to parse -- the score silently degraded, and once +/// `LtmEquation` began parsing each arm eagerly it also tripped an +/// augmentation-bug `debug_assert` on a VALID model. `quote_ident` now also +/// consults `ast::needs_quoting`, the leading-character rule the `print_eqn` +/// path already uses, so the two spellings of one name in a single generated +/// equation agree. +/// +/// This asserts the real property -- the generated arm has a parsed AST -- not +/// merely that the text contains a quote, so it fails for ANY unparseable +/// generated equation rather than only this spelling. +#[test] +fn link_score_quotes_a_canonical_name_that_cannot_be_bare() { + let project = TestProject::new("digit_leading_ident") + .stock("1stock", "0", &["inflow"], &[], None) + .flow("inflow", "\"1stock\" * 0.1", None) + .build_datamodel(); + + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, &project); + let model = sync.models["main"].source; + + // The `1stock -> inflow` edge: the guard form spells both endpoints. + let link_id = LtmLinkId::new(&db, "1stock".to_string(), "inflow".to_string()); + let scored = link_score_equation_text_shaped(&db, link_id, RefShape::Bare, model, sync.project); + let ShapedLinkScore::Scored(lsv) = scored else { + panic!("the 1stock -> inflow link score should be scored, got: {scored:?}"); + }; + + let crate::db::LtmEquation::Scalar(arm) = &lsv.equation else { + panic!( + "a scalar target's link score should be Scalar, got: {:?}", + lsv.name + ); + }; + assert!( + arm.text.contains("\"1stock\""), + "a name that cannot be bare must be quoted in the generated text, got: {}", + arm.text + ); + assert!( + arm.expr.is_some(), + "the generated link-score equation must PARSE (an unparseable arm carries no \ + AST, compiles to no bytecode, and silently zeroes the score); text was: {}", + arm.text + ); +} + /// Build a `StitchPetal<&str>` from `[agg, x1, ..., xm]`. fn petal<'a>(nodes: &[&'a str]) -> super::StitchPetal<&'a str> { super::StitchPetal { diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 1c5bfeb4d..7acb03400 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -2999,10 +2999,30 @@ fn live_reducer_text_for_agg<'a>( .map(|(text, _)| text.as_str()) } -/// Quote an identifier for use in an equation string. -/// Identifiers with special characters (like $, ⁚) need double quotes. +/// Quote a canonical identifier for use in a generated equation string. +/// Identifiers the equation lexer cannot read bare need double quotes: special +/// characters (`$`, `⁚`, `·`) and ALSO a name whose first character cannot START +/// an identifier (`1stock` -- a legal quoted XMILE name, but bare the lexer +/// reads the number `1` then the identifier `stock`). +/// +/// The leading-character rule is [`crate::ast::needs_quoting`], the same +/// predicate the `print_eqn` path's `print_ident` uses -- so the two spellings +/// of one name inside a single generated equation cannot disagree. They did: +/// this used to test only "every char is alphanumeric or `_`", which a leading +/// digit satisfies, so a guard form emitted `print_eqn`'s quoted `"1stock"` in +/// the partial beside a bare `1stock` in the `SIGN(...)` factor -- an +/// unparseable equation, hence a silently-zeroed link score on a valid model. +/// +/// This deliberately stays MORE conservative than `print_ident` rather than +/// delegating outright: `·` (U+00B7) IS `XID_Continue`, so `needs_quoting` +/// alone would spell a module-output composite bare (`mod·out1`). That parses, +/// but it would rewrite the emitted text of every module-composite link score +/// -- a behavior change with no bug behind it. So a name is bare only when BOTH +/// predicates allow it; the conjunction can only ever add quotes. pub(crate) fn quote_ident(ident: &str) -> String { - if ident.chars().all(|c| c.is_alphanumeric() || c == '_') { + let bare_spellable = + ident.chars().all(|c| c.is_alphanumeric() || c == '_') && !crate::ast::needs_quoting(ident); + if bare_spellable { ident.to_string() } else { format!("\"{ident}\"") From b0a2a6f0fd16d4687ea8fc382a55cd1c6e6f8b3b Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 17:20:41 -0700 Subject: [PATCH 10/14] engine: make LTM arm parse failures loud, index occurrences by slot Three findings from a review pass over the typed-equation boundary. Arrayed arm parse errors were silently dropped, and this completes the degradation story 17d4e7c0 claimed. That commit deleted a debug_assert on the grounds that "no AST means no bytecode, so model_ltm_fragment_diagnostics warns". True for a scalar equation; FALSE for an arrayed one. LtmArm::new collapsed a parse failure into a bare expr: None, to_flow_ast's slot map dropped it, and any sibling arm that parsed kept the fragment alive with bytecode -- so the compiler zero-filled the missing slot and no diagnostic fired at all. A silent per-element zero: the exact class this track exists to remove, reintroduced at the new boundary. LtmArm now retains the parse error, which is what distinguishes a FAILED arm from a legitimately EMPTY one (the latter must still drop, as variable::parse_equation drops it), and to_flow_ast rejects the whole equation before any shape-specific assembly. The pin covers both shapes and both mutation directions. Only the first error is kept: the diagnostic names the variable and its text, not a parse position, so a Vec inflated LtmSyntheticVar past clippy's large_enum_variant threshold for data nothing reads. Occurrence lookup was quadratic in an arrayed target's element count: for_slot rescanned the whole occurrence stream once per slot and allocated N temporary vectors. SlotOccurrences groups once and indexes per slot, and is now the ONLY way to obtain an OccurrenceLookup -- the lookup borrows from the index, so the type system forces callers to hoist it out of their per-element loop instead of rebuilding it inside. Measured, because the win is not where the finding implied. Timing model_ltm_variables on a width-N per-element target (the checked-in #[ignore]d harness per_element_generation_scaling): N=50 37.7ms -> 37.3ms N=200 564ms -> 568ms N=100 141ms -> 141ms N=400 2.317s -> 2.304s About 2%, still 4x per doubling. The rescan was a genuine asymptotic term and is gone, but a LARGER pre-existing quadratic dominates: this shape emits N+1 link scores each arrayed over the target's N+1 slots, so (N+1)^2 equation arms per edge -- 160,401 at N=400. The queries it reads are all linear and total ~7ms of that 2.3s, so the cost is the emission shape, not classification. Filed separately; not this branch's to fix, and the harness prints the arm counts that show it. A SiteId child index could overflow on a variadic builtin. Builtin arity is the one child count the AST shape does not bound, so a 65,536-argument MEAN panicked in debug and WRAPPED in release -- re-using an earlier child's SiteId, which makes the wrap's path lookup return a different occurrence and freeze the wrong reference: a plausible wrong score, silent. site_child_index returns None past the addressable range rather than a wrapped value, and the walker records no occurrence for such a child, which routes into the existing loud path (a live-source subscript with no occurrence sets missing_occurrence, so the partial is abandoned and the emitter warns). The per-edge ReferenceSite view still gets its site, so no causal edge is lost. Widening the path element to u32 was rejected: every occurrence carries a boxed path in a salsa-cached per-model IR, so it would double that footprint on every model to serve an arity no real model reaches, and it would move the boundary rather than remove it. Pinned on the boundary function, not on a 65,536-argument fixture that would strain the suite's wall-clock cap. The occurrence read side (SlotOccurrences / OccurrenceLookup) moves to its own file: ltm_augment.rs had reached the 6000-line cap, and this is the coherent unit to extract. --- src/simlin-engine/src/db/ltm/equation.rs | 102 +++++++-- src/simlin-engine/src/db/ltm/link_scores.rs | 34 +-- src/simlin-engine/src/db/ltm_ir.rs | 45 +++- src/simlin-engine/src/db/ltm_ir_tests.rs | 26 +++ src/simlin-engine/src/db/ltm_tests.rs | 203 ++++++++++++++++++ src/simlin-engine/src/ltm_augment.rs | 143 ++---------- .../src/ltm_augment_occurrence.rs | 175 +++++++++++++++ src/simlin-engine/src/ltm_augment_tests.rs | 79 ++++++- .../src/ltm_augment_wrap_test_support.rs | 1 + 9 files changed, 645 insertions(+), 163 deletions(-) create mode 100644 src/simlin-engine/src/ltm_augment_occurrence.rs diff --git a/src/simlin-engine/src/db/ltm/equation.rs b/src/simlin-engine/src/db/ltm/equation.rs index 149f1cbea..784048b1f 100644 --- a/src/simlin-engine/src/db/ltm/equation.rs +++ b/src/simlin-engine/src/db/ltm/equation.rs @@ -31,11 +31,20 @@ use crate::lexer::LexerType; /// One equation arm: the authoritative parsed AST plus its diagnostic text. /// -/// See the module docs for why both are carried. `expr` is `None` for an -/// empty equation (`Expr0::new("")` yields `Ok(None)`, e.g. a discovery-only -/// stub) or the effectively-unreachable case of a generated equation that -/// fails to parse -- both compile to no bytecode, mirroring how the old text -/// path handled an empty/bad `datamodel::Equation`. +/// See the module docs for why both are carried. `expr` is `None` in two +/// materially DIFFERENT cases, which [`LtmArm::parse_error`] distinguishes: +/// +/// - an **empty** equation (`Expr0::new("")` yields `Ok(None)`, e.g. a +/// discovery-only stub) -- legitimate, no errors, and dropped from an +/// `Arrayed` slot map exactly as `variable::parse_equation` drops it; +/// - a **failed parse** -- `parse_error` is `Some`, and [`LtmEquation::to_flow_ast`] +/// surfaces it so the fragment is REJECTED rather than silently +/// zero-filled. +/// +/// Conflating the two is what made a bad arm silent: with siblings that parse, +/// an `Arrayed` equation still produced bytecode, so the "no bytecode ⇒ +/// `model_ltm_fragment_diagnostics` warns" path never fired and that element's +/// score read a constant 0 with no diagnostic at all. #[cfg_attr(feature = "debug-derive", derive(Debug))] #[derive(Clone, PartialEq, salsa::Update)] pub struct LtmArm { @@ -44,18 +53,29 @@ pub struct LtmArm { pub text: String, /// The authoritative compiled AST (`Expr0::new(text)`). pub expr: Option, + /// `Some` iff `text` FAILED to parse -- never merely because it was empty. + /// Preserved (rather than discarded at construction) so the arm that failed + /// can reject its whole equation; see the type docs. + /// + /// The FIRST error only, deliberately: this is a boolean-with-provenance, + /// since the emitted diagnostic names the variable and its text rather than + /// the parse position, and the whole equation is rejected regardless of how + /// many arms or positions failed. Keeping a `Vec` here inflated + /// `LtmSyntheticVar` (hence `ShapedLinkScore`) past clippy's + /// `large_enum_variant` threshold for data no consumer reads. + pub parse_error: Option, } impl LtmArm { /// Parse `text` into the authoritative AST once, at the generation /// (source-format) boundary. pub fn new(text: String) -> Self { - // Degrade exactly as the old text path did: an unparseable equation - // carries no AST and compiles to no bytecode, so `compile_phase` sees an - // empty expr list, `flow_bytecodes` stays `None`, and - // `model_ltm_fragment_diagnostics` warns that the variable keeps a - // layout slot with no bytecode. That warning IS the loud path -- there - // is deliberately no assertion here. + // Degradation is non-fatal but LOUD: the parse errors are RETAINED and + // `to_flow_ast` returns them, so the fragment is rejected and + // `model_ltm_fragment_diagnostics` warns. Retaining them (rather than + // collapsing the failure into a bare `expr: None`) is what makes the + // ARRAYED case loud too -- with siblings that parse, a dropped arm would + // leave the fragment alive and that element's score reading a silent 0. // // An earlier revision did `debug_assert!(false, ..)`, on the theory that // a generated equation is always a `print_eqn` re-print and so a parse @@ -66,10 +86,19 @@ impl LtmArm { // (it now shares `ast::needs_quoting` with `print_ident`), but the // degradation stays non-fatal on principle -- this runs inside a salsa // query on the ordinary read path, where aborting on user input is - // strictly worse than the warning, and libsimlin release builds are + // strictly worse than a diagnostic, and libsimlin release builds are // panic=abort. - let expr = Expr0::new(&text, LexerType::Equation).unwrap_or_default(); - Self { text, expr } + let (expr, parse_error) = match Expr0::new(&text, LexerType::Equation) { + Ok(expr) => (expr, None), + // `Expr0::new` reports every position it found; keep the first as + // the failure's provenance (see the field docs). + Err(errs) => (None, errs.into_iter().next()), + }; + Self { + text, + expr, + parse_error, + } } } @@ -236,6 +265,26 @@ impl LtmEquation { } } + /// Every arm's retained parse errors, in arm order (elements then default). + /// Non-empty means at least one arm's generated text failed to parse -- an + /// augmentation-layer bug -- as distinct from an arm that is legitimately + /// EMPTY (which carries no error). + fn arm_parse_errors(&self) -> Vec { + let arms: Vec<&LtmArm> = match self { + LtmEquation::Scalar(arm) | LtmEquation::ApplyToAll(_, arm) => vec![arm], + LtmEquation::Arrayed { + elements, default, .. + } => elements + .iter() + .map(|(_, arm)| arm) + .chain(default.as_ref()) + .collect(), + }; + arms.iter() + .filter_map(|arm| arm.parse_error.clone()) + .collect() + } + /// Build the flow-phase `Ast` for compilation and the layout /// implicit-var scan, resolving the dimension NAMES to `Dimension`s against /// the project datamodel dims. Mirrors `variable::parse_equation` MINUS the @@ -243,13 +292,29 @@ impl LtmEquation { /// dimension-resolution error alongside (so the compiler drops the fragment /// and the diagnostic pass warns, exactly as a text parse error did). /// - /// `None` ast is an empty/unparseable equation (no arm expr): it compiles - /// to no bytecode. LTM synthetic variables are flow-phase only, so there is - /// no init-phase ast to build. + /// A FAILED arm parse is returned as an error and yields NO ast, for every + /// shape. That is load-bearing for `Arrayed`: dropping the bad arm and + /// keeping its parsing siblings (which is what `variable::parse_equation` + /// does for an EMPTY arm) would leave the fragment with bytecode, so the + /// compiler would zero-fill the missing slot, `flow_bytecodes` would stay + /// `Some`, and `model_ltm_fragment_diagnostics` would emit nothing -- a + /// silent per-element zero. An arm that is legitimately EMPTY still just + /// drops, exactly as `parse_equation` drops it. + /// + /// `None` ast with no errors is an empty equation: it compiles to no + /// bytecode. LTM synthetic variables are flow-phase only, so there is no + /// init-phase ast to build. pub fn to_flow_ast( &self, dimensions: &[datamodel::Dimension], ) -> (Option>, Vec) { + // A generated arm that failed to parse rejects the whole equation, + // BEFORE any shape-specific assembly -- see the note above on why the + // `Arrayed` slot map must not simply drop it. + let parse_errors = self.arm_parse_errors(); + if !parse_errors.is_empty() { + return (None, parse_errors); + } match self { LtmEquation::Scalar(arm) => (arm.expr.clone().map(Ast::Scalar), vec![]), LtmEquation::ApplyToAll(dims, arm) => { @@ -268,7 +333,8 @@ impl LtmEquation { has_except_default, } => { // Mirror `parse_equation`'s Arrayed arm: drop element slots with - // no expr (an empty/unparseable body) rather than error. + // no expr. Reaching here means no arm FAILED to parse, so a + // missing expr is an empty body only. let map: HashMap = elements .iter() .filter_map(|(subscript, arm)| { diff --git a/src/simlin-engine/src/db/ltm/link_scores.rs b/src/simlin-engine/src/db/ltm/link_scores.rs index bb19b05e1..aa063f706 100644 --- a/src/simlin-engine/src/db/ltm/link_scores.rs +++ b/src/simlin-engine/src/db/ltm/link_scores.rs @@ -1242,6 +1242,10 @@ pub(super) fn try_scalar_to_arrayed_link_scores( .map(Vec::as_slice) .unwrap_or(&[]); let slot_map = ArrayedSlotMap::new(ast); + // Group the target's occurrences by slot ONCE (see `SlotOccurrences`): + // the per-element loops below index it per slot instead of rescanning + // the whole stream, which was quadratic in the element count. + let slot_occurrences = crate::ltm_augment::SlotOccurrences::new(to_occurrences); // Build one `LtmSyntheticVar` for `element` from its equation text and // that text's dependency set. The element name is the only part of the @@ -1283,10 +1287,7 @@ pub(super) fn try_scalar_to_arrayed_link_scores( // as `apply_implicit_with_lookup` leaves its value unwrapped. let gf_table_ref = slot_refs.for_element(&crate::common::CanonicalElementName::from_raw(element)); - let occ = crate::ltm_augment::OccurrenceLookup::for_slot( - to_occurrences, - slot_map.slot_for(element), - ); + let occ = slot_occurrences.for_slot(slot_map.slot_for(element)); match crate::ltm_augment::generate_scalar_to_element_equation( from, to, @@ -2404,6 +2405,10 @@ fn emit_per_element_link_scores( .map(Vec::as_slice) .unwrap_or(&[]); let slot_map = ArrayedSlotMap::new(ast); + // Group the target's occurrences by slot ONCE (see `SlotOccurrences`): + // the per-element loops below index it per slot instead of rescanning + // the whole stream, which was quadratic in the element count. + let slot_occurrences = crate::ltm_augment::SlotOccurrences::new(to_occurrences); // Arrayed deps sharing a target dim get element-pinned in the scalar // per-element equation (mirroring `try_scalar_to_arrayed_link_scores`); @@ -2539,10 +2544,7 @@ fn emit_per_element_link_scores( if !emitted.insert(name.clone()) { continue; } - let occ = crate::ltm_augment::OccurrenceLookup::for_slot( - to_occurrences, - slot_map.slot_for(element), - ); + let occ = slot_occurrences.for_slot(slot_map.slot_for(element)); match crate::ltm_augment::generate_per_element_link_equation( from, to, @@ -3190,6 +3192,10 @@ pub(super) fn emit_agg_to_target_link_scores( .map(Vec::as_slice) .unwrap_or(&[]); let slot_map = ArrayedSlotMap::new(ast); + // Group the target's occurrences by slot ONCE (see `SlotOccurrences`): + // the per-element loops below index it per slot instead of rescanning + // the whole stream, which was quadratic in the element count. + let slot_occurrences = crate::ltm_augment::SlotOccurrences::new(to_occurrences); // GH #910: when `to` is an implicit WITH-LOOKUP variable, the per-element // partials below are full re-evaluations of its RAW element equation (with // the reducer substituted by the agg name), while the guard form ratios them @@ -3459,7 +3465,7 @@ pub(super) fn emit_agg_to_target_link_scores( // Agg source: the live thing is the synthetic agg, never a // recorded occurrence, so the real stream is behavior-neutral // (the wrap's agg lookups all miss). A scalar target is slot 0. - &crate::ltm_augment::OccurrenceLookup::for_slot(to_occurrences, 0), + &slot_occurrences.for_slot(0), ) { Ok(equation) => vars.push(LtmSyntheticVar { name, @@ -3525,10 +3531,7 @@ pub(super) fn emit_agg_to_target_link_scores( // Agg source: the live thing is the synthetic agg, never a // recorded occurrence, so the real stream is behavior-neutral // (the wrap's agg lookups all miss). A2A body is slot 0. - &crate::ltm_augment::OccurrenceLookup::for_slot( - to_occurrences, - slot_map.slot_for(element), - ), + &slot_occurrences.for_slot(slot_map.slot_for(element)), ) { Ok(equation) => edge_vars.push(LtmSyntheticVar { name, @@ -3607,10 +3610,7 @@ pub(super) fn emit_agg_to_target_link_scores( // Agg source: the live thing is the synthetic agg, // never a recorded occurrence, so the real stream is // behavior-neutral (the wrap's agg lookups all miss). - &crate::ltm_augment::OccurrenceLookup::for_slot( - to_occurrences, - slot_map.slot_for(element), - ), + &slot_occurrences.for_slot(slot_map.slot_for(element)), ) } }; diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index 05012dde8..ac6d8b667 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -565,6 +565,38 @@ fn classify_occurrence_axes( .collect() } +/// The [`SiteId`] child index for a builtin's `n`-th content, or `None` when the +/// arity exceeds what one path element can address. +/// +/// A `SiteId` element is a `u16`, which every AST-shaped child count fits by +/// construction (`Op1`/`Op2`/`If`/`Subscript` arity is bounded by the node). +/// Builtin arity is NOT: `MEAN`/`SUM` and friends are variadic, so a call with +/// 65,536+ arguments is representable in the AST. +/// +/// Returning `None` -- rather than wrapping -- is the whole point. A wrapped +/// index would RE-USE an earlier child's `SiteId`, so the wrap's path lookup +/// would silently return a DIFFERENT occurrence: if the two children carry +/// different subscript shapes, the wrap holds or freezes the wrong reference and +/// emits a plausible, wrong link score. That is silent corruption, the failure +/// class this IR exists to eliminate. +/// +/// The caller records NO occurrence for an unaddressable child. That is the loud +/// outcome, because it routes into machinery that already exists: a live-source +/// subscript with no occurrence at its tracked path (on a non-empty stream) sets +/// `WrapOutcome::missing_occurrence`, so the partial is abandoned and the +/// db-bearing emitter warns and skips. The per-EDGE `ReferenceSite` view is +/// unaffected (the caller still pushes those), so the causal graph keeps every +/// edge -- only the SiteId-addressable occurrence view declines to name a child +/// it cannot address. +/// +/// Widening the path element to `u32` was the alternative. It is rejected: the +/// occurrence IR is salsa-cached per model and every occurrence carries a boxed +/// path, so widening doubles that footprint on every model to serve an arity no +/// real model reaches -- and it would not remove the case, only move it. +fn site_child_index(n: usize) -> Option { + u16::try_from(n).ok() +} + /// `true` iff `builtin` is `PREVIOUS(...)` / `INIT(...)`: everything inside is /// already lagged (read at t-1) or frozen (read at t=0). Used to set the /// `already_lagged` occurrence marker so the transform does not re-wrap it. @@ -773,8 +805,17 @@ fn walk_all_in_expr( } // Contents of a PREVIOUS/INIT call are already lagged/frozen. let child_lagged = already_lagged || builtin_is_previous_or_init(builtin); - let mut child: u16 = 0; + // Builtin arity is the one child count not bounded by the AST shape + // (`MEAN(a, b, ...)` is variadic), so it is the only place a `SiteId` + // element can run out of range. `site_child_index` returns `None` + // past the addressable range and we then record NO occurrence for + // that child -- see its rustdoc for why that is the loud outcome. + let mut child_n: usize = 0; walk_builtin_expr(builtin, |contents| { + let Some(child) = site_child_index(child_n) else { + child_n += 1; + return; + }; acc.path.push(child); match contents { BuiltinContents::Ident(id, _) => { @@ -823,7 +864,7 @@ fn walk_all_in_expr( BuiltinContents::LookupTable(_) => {} } acc.path.pop(); - child += 1; + child_n += 1; }); if pushed_reducer_key { reducer_keys.pop(); diff --git a/src/simlin-engine/src/db/ltm_ir_tests.rs b/src/simlin-engine/src/db/ltm_ir_tests.rs index a5c0e4f95..7cd6c8729 100644 --- a/src/simlin-engine/src/db/ltm_ir_tests.rs +++ b/src/simlin-engine/src/db/ltm_ir_tests.rs @@ -1668,3 +1668,29 @@ mod occurrence_ir_tests { ); } } + +/// F3: a `SiteId` child index must never WRAP. The addressable range is pinned +/// at its exact boundary, so a variadic builtin's 65,536th content declines to +/// be addressed rather than re-using child 0's path. +/// +/// Pinned on the pure boundary function rather than by building a 65,536-argument +/// `MEAN` call: the fixture would be enormous and slow (the 3-minute suite cap), +/// while the boundary is the entire property. The consequence of getting this +/// wrong is silent -- a wrapped index makes the wrap's lookup return a different +/// occurrence, so it holds or freezes the wrong reference and emits a plausible +/// wrong score -- which is why the guard returns `None` instead of a wrapped +/// value, and why the walker records no occurrence for such a child. +#[test] +fn site_child_index_declines_rather_than_wrapping() { + use super::site_child_index; + + assert_eq!(site_child_index(0), Some(0)); + assert_eq!(site_child_index(1), Some(1)); + // The last addressable child. + assert_eq!(site_child_index(65_535), Some(u16::MAX)); + // One past it must DECLINE, not wrap to 0 (which would collide with the + // first child's SiteId and silently mis-address the occurrence). + assert_eq!(site_child_index(65_536), None); + assert_eq!(site_child_index(65_537), None); + assert_eq!(site_child_index(usize::MAX), None); +} diff --git a/src/simlin-engine/src/db/ltm_tests.rs b/src/simlin-engine/src/db/ltm_tests.rs index be8e76fb4..c6af3e3bd 100644 --- a/src/simlin-engine/src/db/ltm_tests.rs +++ b/src/simlin-engine/src/db/ltm_tests.rs @@ -620,6 +620,209 @@ fn link_score_quotes_a_canonical_name_that_cannot_be_bare() { ); } +/// An unparseable generated arm degrades LOUDLY and WITHOUT PANICKING, for the +/// scalar shape AND for an `Arrayed` equation whose OTHER arms parse fine. +/// +/// This pins the fallback deliberately kept in place of the deleted +/// `debug_assert!`. Two properties, and the arrayed one is the sharp edge: +/// +/// - the arm retains its parse errors (so `expr` being `None` is +/// distinguishable from a legitimately EMPTY arm, which must still drop +/// silently); +/// - `to_flow_ast` therefore yields NO ast and surfaces the errors, so +/// `compile_ltm_equation_fragment` returns `None` -- the condition +/// `model_ltm_fragment_diagnostics` turns into its "keeps a layout slot but +/// no bytecode" Warning. +/// +/// Before this, `LtmArm::new` collapsed a parse failure into a bare +/// `expr: None` and the `Arrayed` slot map silently dropped it. With siblings +/// that parsed, the fragment still had bytecode, so the compiler zero-filled +/// the missing slot and NO diagnostic fired: a silent per-element zero, the +/// exact defect class the typed-equation work exists to remove. The scalar case +/// was loud only incidentally -- it had no surviving sibling to keep the +/// fragment alive. +/// +/// Constructing the degenerate value is legitimate through the public API: +/// `LtmArm::new` takes an arbitrary `String`, so there is no "cannot be built" +/// escape hatch here. +#[test] +fn unparseable_generated_arm_degrades_loudly_without_panicking() { + use crate::db::{LtmArm, LtmEquation}; + + // Unparseable for the same reason the `1stock` bug was: a bare leading + // digit lexes as a number followed by an identifier. + let bad = "1stock * 0.1"; + + // (a) The arm itself: no panic, no AST, errors RETAINED. + let arm = LtmArm::new(bad.to_string()); + assert!(arm.expr.is_none(), "unparseable text must yield no AST"); + assert!( + arm.parse_error.is_some(), + "the parse error must be retained -- discarding it is what made the \ + arrayed case silent" + ); + // An EMPTY arm is the legitimate `expr == None` and must NOT carry errors. + let empty = LtmArm::new(String::new()); + assert!(empty.expr.is_none() && empty.parse_error.is_none()); + + let project = TestProject::new("bad_arm_degradation") + .named_dimension("Region", &["nyc", "boston"]) + .array_aux("pop[Region]", "10") + .aux("other", "1", None) + .build_datamodel(); + let db = SimlinDb::default(); + let sync = sync_from_datamodel(&db, &project); + let model = sync.models["main"].source; + let dims = crate::db::project_datamodel_dims(&db, sync.project); + + // (b) Scalar: no ast, errors surfaced, fragment rejected. + let scalar = LtmEquation::scalar(bad.to_string()); + let (ast, errs) = scalar.to_flow_ast(dims); + assert!( + ast.is_none() && !errs.is_empty(), + "scalar must reject loudly" + ); + assert!( + compile_ltm_equation_fragment(&db, "$⁚ltm⁚bad⁚scalar", &scalar, model, sync.project) + .is_none(), + "a rejected equation must not produce a fragment (the diagnostic pass \ + reports the missing bytecode)" + ); + + // (c) Arrayed with ONE bad arm and one GOOD sibling -- the case that was + // silent. The whole equation must be rejected, not zero-filled. + let arrayed = LtmEquation::arrayed( + vec!["Region".to_string()], + vec![ + ("nyc".to_string(), bad.to_string()), + ("boston".to_string(), "other * 2".to_string()), + ], + None, + false, + ); + let (ast, errs) = arrayed.to_flow_ast(dims); + assert!( + ast.is_none() && !errs.is_empty(), + "one unparseable arm must reject the whole arrayed equation rather than \ + drop the slot and let the surviving sibling keep the fragment alive" + ); + assert!( + compile_ltm_equation_fragment(&db, "$⁚ltm⁚bad⁚arrayed", &arrayed, model, sync.project) + .is_none(), + "the arrayed fragment must be rejected -- otherwise the missing slot is \ + zero-filled and no diagnostic fires" + ); + + // (d) An arrayed equation whose arms ALL parse is unaffected: an empty + // default still drops silently and the fragment compiles. + let good = LtmEquation::arrayed( + vec!["Region".to_string()], + vec![ + ("nyc".to_string(), "other * 2".to_string()), + ("boston".to_string(), "other * 3".to_string()), + ], + Some(String::new()), + false, + ); + let (ast, errs) = good.to_flow_ast(dims); + assert!( + ast.is_some() && errs.is_empty(), + "an all-parsing arrayed equation with an EMPTY default must still build" + ); +} + +/// Reproducible timing harness for per-element LTM generation over a WIDE +/// dimension -- the shape whose occurrence lookup was quadratic. +/// +/// `build_arrayed_link_score_equation` wraps each of an `Ast::Arrayed` target's +/// N element equations separately, and each wrap needs that slot's occurrence +/// stream. When `OccurrenceLookup::for_slot` rescanned the target's WHOLE +/// stream per slot, that was Theta(N^2) comparisons plus N temporary vectors. +/// +/// `#[ignore]`d: this is the measuring instrument, not a gate. Checked in +/// (rather than the numbers) so the measurement is reproducible -- run with +/// `cargo test -p simlin-engine --release --lib -- --ignored --nocapture +/// per_element_generation_scaling`. A timing assertion in the default suite +/// would be flaky and would risk the 3-minute wall-clock cap; the structural +/// guarantee is instead pinned by `slot_occurrence_index_groups_every_slot`. +#[test] +#[ignore] +fn per_element_generation_scaling() { + use std::time::Instant; + + fn generate_for_width(n: usize) -> std::time::Duration { + let elems: Vec = (0..n).map(|i| format!("e{i}")).collect(); + let elem_refs: Vec<&str> = elems.iter().map(String::as_str).collect(); + // Each element equation reads its OWN element of the source plus a + // shared scalar, so every slot carries several occurrences. + let eqns: Vec<(String, String)> = elems + .iter() + .map(|e| (e.clone(), format!("pop[{e}] * rate * 0.01"))) + .collect(); + let eqn_refs: Vec<(&str, &str)> = + eqns.iter().map(|(e, q)| (e.as_str(), q.as_str())).collect(); + + let project = TestProject::new("wide_per_element") + .named_dimension("Wide", &elem_refs) + .aux("rate", "1", None) + .array_flow_with_ranges("growth[Wide]", eqn_refs) + .array_stock("pop[Wide]", "10", &["growth"], &[], None) + .build_datamodel(); + + let mut db = SimlinDb::default(); + let (source_project, model) = { + let sync = sync_from_datamodel(&db, &project); + (sync.project, sync.models["main"].source) + }; + use salsa::Setter; + source_project.set_ltm_enabled(&mut db).to(true); + + // Sub-phase timings: which query actually scales quadratically. + let t0 = Instant::now(); + let _sites = crate::db::ltm_ir::model_ltm_reference_sites(&db, model, source_project); + let t_sites = t0.elapsed(); + let t0 = Instant::now(); + let _edges = crate::db::model_element_causal_edges(&db, model, source_project); + let t_edges = t0.elapsed(); + let t0 = Instant::now(); + let _circuits = crate::db::model_loop_circuits_tiered(&db, model, source_project); + let t_circuits = t0.elapsed(); + + let start = Instant::now(); + let ltm = crate::db::model_ltm_variables(&db, model, source_project); + let elapsed = start.elapsed(); + let n_scores = ltm + .vars + .iter() + .filter(|v| v.name.contains("link_score")) + .count(); + let n_arms: usize = ltm + .vars + .iter() + .map(|v| match &v.equation { + crate::db::LtmEquation::Arrayed { elements, .. } => elements.len(), + _ => 1, + }) + .sum(); + println!( + " sub-phases: ref_sites {t_sites:?}, element_edges {t_edges:?}, \ + tiered_circuits {t_circuits:?}; link_scores {n_scores}, total arms {n_arms}" + ); + // Keep the work observable so nothing is optimized away, and confirm + // the fixture actually generated per-element scores. + assert!( + ltm.vars.iter().any(|v| v.name.contains("link_score")), + "fixture must generate link scores" + ); + elapsed + } + + for n in [50usize, 100, 200, 400] { + let d = generate_for_width(n); + println!("width {n:>4}: model_ltm_variables {d:?}"); + } +} + /// Build a `StitchPetal<&str>` from `[agg, x1, ..., xm]`. fn petal<'a>(nodes: &[&'a str]) -> super::StitchPetal<&'a str> { super::StitchPetal { diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 7acb03400..01542ea7b 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -22,129 +22,17 @@ use std::collections::{HashMap, HashSet}; use crate::db::LtmEquation; use crate::db::RefShape; use crate::db::ltm_ir::{ - OccurrenceAxis, OccurrenceRef, OccurrenceSite, OtherDepVerdict, derive_other_dep_verdict, + OccurrenceAxis, OccurrenceSite, OtherDepVerdict, derive_other_dep_verdict, }; -/// The ceteris-paribus wrap's view of the occurrence IR for ONE slot of a -/// target equation: the per-occurrence access shape and per-axis -/// classification `db::ltm_ir` already decided on the target's `Expr2` AST, -/// keyed by the structural child-index path with the slot prefix stripped. -/// -/// This is the SINGLE classifier family the wrap consumes. The wrap runs on -/// the target's printed-and-reparsed `Expr0`; rather than re-deriving each -/// occurrence's shape on that `Expr0` (the retired Expr0 mirror classifiers), -/// it tracks the same left-to-right child-index path `db::ltm_ir::walk_all_in_expr` -/// builds and looks the occurrence up here. The print->reparse round trip is -/// child-index-isomorphic to the `Expr2` walk (proved corpus-wide by -/// `classifier_agreement_tests::assert_occurrence_stream_aligns`), so a path -/// hit returns exactly the shape/axes the edge emitter used -- the two -/// families cannot drift because there is only one. -/// -/// A path MISS on a live-source subscript is the one residual production hazard -/// -- a novel shape the alignment gate cannot cover could make the reparse -/// non-isomorphic. That is NOT silently tolerated: the subscript arm flags it -/// (`WrapOutcome::missing_occurrence`) on a non-empty stream, so the partial is -/// abandoned with a loud skip-and-warn instead of freezing the live reference -/// into a constant-0 score. -/// -/// `for_slot` rebases to slot-local paths because the wrap walks a single -/// slot's expression from its root: an `Ast::Scalar`/`ApplyToAll` target is -/// slot `0`; an `Ast::Arrayed` target's per-element slots are numbered in -/// canonical element-key-sorted order (`build_arrayed_link_score_equation` -/// wraps each slot separately), matching the SiteId slot prefix. -pub(crate) struct OccurrenceLookup<'a> { - /// `(slot-local path, occurrence)` for every occurrence in this slot, in - /// document order. LTM equations are short, so a linear scan is cheaper - /// than hashing and keeps the borrow lifetimes trivial. - entries: Vec<(&'a [u16], &'a OccurrenceSite)>, -} - -impl<'a> OccurrenceLookup<'a> { - /// Build the lookup for `slot` of `occs` (the target's whole occurrence - /// stream), rebasing each occurrence's SiteId to its slot-local path. - pub(crate) fn for_slot(occs: &'a [OccurrenceSite], slot: u16) -> Self { - let entries = occs - .iter() - .filter(|o| o.site_id.0.first() == Some(&slot)) - .map(|o| (&o.site_id.0[1..], o)) - .collect(); - OccurrenceLookup { entries } - } - - /// The occurrence at exactly `path`, if any (a genuine causal reference at - /// that node). `None` for a node that is not a recorded occurrence (a - /// function name, a dimension name, a literal element selector, or a - /// deeper index the walk skipped). - fn get(&self, path: &[u16]) -> Option<&'a OccurrenceSite> { - self.entries - .iter() - .find(|(p, _)| *p == path) - .map(|(_, o)| *o) - } - - /// Whether this slot recorded NO occurrences. `true` for a genuinely - /// source-free slot -- an EXCEPT default that never references `from`, whose - /// trivial-zero guard form is legitimate, or an AGG-source generator whose - /// live source (the synthetic agg) is never a recorded occurrence. A miss on - /// a NON-empty lookup, by contrast, is a walker desync -- the `missing_occurrence` - /// guard in the subscript arm keys on this so it never false-fires on a - /// legitimately source-free slot. - fn is_empty(&self) -> bool { - self.entries.is_empty() - } +/// The wrap's read side of the occurrence IR ([`SlotOccurrences`] / +/// [`OccurrenceLookup`]), in its own file only to keep this one under the +/// project line-count lint. Re-exported so callers keep naming them +/// `crate::ltm_augment::*`. +#[path = "ltm_augment_occurrence.rs"] +mod occurrence; - /// Does any occurrence of the live source (a model `Variable` or a - /// `ModuleOutput` composite) STRICTLY under `prefix` carry access shape - /// `shape`? This is the occurrence-IR form of the retired - /// `expr0_contains_live_match` lookahead: an array-reducer `App` at - /// `prefix` is frozen whole (GH #517) unless it genuinely holds the live - /// reference, which is exactly "a live-shaped occurrence lives in its - /// subtree". - /// - /// Index-nested occurrences (`other_arr[live]`) are EXCLUDED: the retired - /// `expr0_contains_live_match` only inspected subscript *heads*, never - /// recursing into a subscript's index expressions, so an index-nested - /// `live` never made the enclosing reducer "hold the live ref". The - /// occurrence IR marks those `index_nested`, so filtering on it reproduces - /// that boundary exactly (the reducer freezes whole and the bare `live` - /// stays live -- `char_reducer_index_nested_freeze`). - /// - /// A `live` source can be either a model `Variable` (the ordinary link - /// score) or a `module·port` composite (`OccurrenceRef::ModuleOutput`, the - /// `db::module_link_score_equation` live channel). The retired - /// `expr0_contains_live_match` matched on the bare-`Var` ident text and so - /// treated BOTH the same -- a composite read bare inside a reducer made the - /// reducer hold the live ref. Matching on the occurrence's resolved name - /// (whichever variant) reproduces that: a module-output live source inside a - /// reducer recurses just like a variable one, instead of freezing whole. - fn subtree_has_live_shape( - &self, - prefix: &[u16], - live: &Ident, - shape: &RefShape, - ) -> bool { - self.entries.iter().any(|(p, o)| { - p.len() > prefix.len() - && p.starts_with(prefix) - && !o.index_nested - && &o.shape == shape - && occurrence_names_source(&o.reference, live) - }) - } -} - -/// Whether occurrence reference `reference` names the live source `live` -- -/// either a model `Variable` or a `module·port` composite -/// (`OccurrenceRef::ModuleOutput`). The wrap treats both alike (a bare read of -/// either is a live match), so the occurrence-IR predicates must too. -fn occurrence_names_source(reference: &OccurrenceRef, live: &Ident) -> bool { - match reference { - OccurrenceRef::Variable(v) => &Ident::::new(v) == live, - OccurrenceRef::ModuleOutput { composite, .. } => { - &Ident::::new(composite) == live - } - } -} +pub(crate) use occurrence::{OccurrenceLookup, SlotOccurrences}; /// The implicit WITH-LOOKUP rules (GH #910), in their own file only to keep this /// one under the project line-count lint. Re-exported below so callers keep @@ -1556,7 +1444,8 @@ pub(crate) fn build_partial_equation_shaped_with_live_ref( source_dim_elements, iter_ctx, ); - let occ = OccurrenceLookup::for_slot(&occurrences, 0); + let slot_occurrences = SlotOccurrences::new(&occurrences); + let occ = slot_occurrences.for_slot(0); let (transformed, out) = wrap_changed_first_ast( equation_text, deps, @@ -3689,6 +3578,10 @@ fn build_arrayed_link_score_equation( .map(String::as_str) .chain(source_dim_names.iter().map(String::as_str)) .collect(); + // Group the target's occurrences by slot ONCE, outside the per-element + // closure below: this is the arrayed target whose N slots made the old + // per-slot rescan quadratic (see `SlotOccurrences`). + let slot_occurrences = SlotOccurrences::new(to_occurrences); // `slot` is the per-element occurrence-stream slot: `walk_all_in_expr` // numbers `Ast::Arrayed` slots in canonical element-key-sorted order (then // the default after the last element), so the wrap for element `slot` @@ -3713,7 +3606,7 @@ fn build_arrayed_link_score_equation( .into_iter() .filter(|d| !source_dim_token_set.contains(d.as_str())) .collect(); - let occ = OccurrenceLookup::for_slot(to_occurrences, slot); + let occ = slot_occurrences.for_slot(slot); shaped_guard_form_text( &elem_eqn_text, &deps_e, @@ -3933,7 +3826,8 @@ fn generate_auxiliary_to_auxiliary_equation( }; // A scalar / `Ast::ApplyToAll` target is a single body -- slot 0 of the // occurrence stream. - let occ = OccurrenceLookup::for_slot(to_occurrences, 0); + let slot_occurrences = SlotOccurrences::new(to_occurrences); + let occ = slot_occurrences.for_slot(0); let text = shaped_guard_form_text( &to_equation, &deps, @@ -4245,7 +4139,8 @@ fn generate_stock_to_flow_equation( // error -- see `source_ref_for_guard` (applied inside // `shaped_guard_form_text`, which also handles the GH #743 // changed-last fallback for an unfreezable changed-first partial). - let occ = OccurrenceLookup::for_slot(to_occurrences, 0); + let slot_occurrences = SlotOccurrences::new(to_occurrences); + let occ = slot_occurrences.for_slot(0); let text = shaped_guard_form_text( &flow_equation, &deps, diff --git a/src/simlin-engine/src/ltm_augment_occurrence.rs b/src/simlin-engine/src/ltm_augment_occurrence.rs new file mode 100644 index 000000000..0dece32a8 --- /dev/null +++ b/src/simlin-engine/src/ltm_augment_occurrence.rs @@ -0,0 +1,175 @@ +// Copyright 2026 The Simlin Authors. All rights reserved. +// Use of this source code is governed by the Apache License, +// Version 2.0, that can be found in the LICENSE file. + +//! The ceteris-paribus wrap's read side of the occurrence IR: +//! [`SlotOccurrences`] (a target's occurrence stream grouped by slot) and +//! [`OccurrenceLookup`] (one slot's view, keyed by structural path). +//! +//! Split out of `ltm_augment.rs` only to keep that file under the project +//! line-count lint; included via `#[path]`, so `super::*` resolves the parent's +//! private items and every caller keeps naming these +//! `crate::ltm_augment::*`. + +use std::collections::HashMap; + +use crate::common::{Canonical, Ident}; +use crate::db::RefShape; +use crate::db::ltm_ir::{OccurrenceRef, OccurrenceSite}; + +/// The ceteris-paribus wrap's view of the occurrence IR for ONE slot of a +/// target equation: the per-occurrence access shape and per-axis +/// classification `db::ltm_ir` already decided on the target's `Expr2` AST, +/// keyed by the structural child-index path with the slot prefix stripped. +/// +/// This is the SINGLE classifier family the wrap consumes. The wrap runs on +/// the target's printed-and-reparsed `Expr0`; rather than re-deriving each +/// occurrence's shape on that `Expr0` (the retired Expr0 mirror classifiers), +/// it tracks the same left-to-right child-index path `db::ltm_ir::walk_all_in_expr` +/// builds and looks the occurrence up here. The print->reparse round trip is +/// child-index-isomorphic to the `Expr2` walk (proved corpus-wide by +/// `classifier_agreement_tests::assert_occurrence_stream_aligns`), so a path +/// hit returns exactly the shape/axes the edge emitter used -- the two +/// families cannot drift because there is only one. +/// +/// A path MISS on a live-source subscript is the one residual production hazard +/// -- a novel shape the alignment gate cannot cover could make the reparse +/// non-isomorphic. That is NOT silently tolerated: the subscript arm flags it +/// (`WrapOutcome::missing_occurrence`) on a non-empty stream, so the partial is +/// abandoned with a loud skip-and-warn instead of freezing the live reference +/// into a constant-0 score. +/// +/// Slot paths are rebased to slot-local form because the wrap walks a single +/// slot's expression from its root: an `Ast::Scalar`/`ApplyToAll` target is +/// slot `0`; an `Ast::Arrayed` target's per-element slots are numbered in +/// canonical element-key-sorted order (`build_arrayed_link_score_equation` +/// wraps each slot separately), matching the SiteId slot prefix. Obtain one from +/// [`SlotOccurrences::for_slot`]. +pub(crate) struct OccurrenceLookup<'a> { + /// `(slot-local path, occurrence)` for every occurrence in this slot, in + /// document order. LTM equations are short, so a linear scan within a slot + /// is cheaper than hashing. + entries: &'a [(&'a [u16], &'a OccurrenceSite)], +} + +/// A target's occurrence stream GROUPED BY SLOT, built in one pass. +/// +/// This exists for the arrayed case: `build_arrayed_link_score_equation` wraps +/// each of N element equations separately and needs that slot's occurrences +/// each time. Filtering the whole stream per slot made generating ONE edge's +/// score Theta(N^2) (plus N temporary vectors) -- measured at 37.7ms / 141ms / +/// 564ms / 2.32s for N = 50 / 100 / 200 / 400, a clean 4x per doubling. Grouping +/// once and indexing per slot makes it linear. +/// +/// It is also the ONLY way to obtain an [`OccurrenceLookup`], deliberately: the +/// lookup borrows from this index, so the type system requires callers to hoist +/// it out of their per-element loop rather than rebuild it inside. (A +/// convenience constructor that built a throwaway index per call would compile +/// only by re-introducing the quadratic.) +pub(crate) struct SlotOccurrences<'a> { + by_slot: HashMap>, +} + +impl<'a> SlotOccurrences<'a> { + /// Group `occs` (a target's whole occurrence stream) by slot, rebasing each + /// occurrence's SiteId to its slot-local path. One pass over the stream. + pub(crate) fn new(occs: &'a [OccurrenceSite]) -> Self { + let mut by_slot: HashMap> = HashMap::new(); + for o in occs { + // An occurrence always carries a slot prefix (`walk_all_in_expr` + // pushes it before descending), so an empty SiteId cannot occur; + // skipping rather than indexing keeps this total either way. + if let Some(&slot) = o.site_id.0.first() { + by_slot + .entry(slot) + .or_default() + .push((&o.site_id.0[1..], o)); + } + } + SlotOccurrences { by_slot } + } + + /// The lookup for `slot`. Empty for a slot with no recorded occurrences + /// (which [`OccurrenceLookup::is_empty`] distinguishes from a desync). + pub(crate) fn for_slot(&self, slot: u16) -> OccurrenceLookup<'_> { + OccurrenceLookup { + entries: self.by_slot.get(&slot).map(Vec::as_slice).unwrap_or(&[]), + } + } +} + +impl<'a> OccurrenceLookup<'a> { + /// The occurrence at exactly `path`, if any (a genuine causal reference at + /// that node). `None` for a node that is not a recorded occurrence (a + /// function name, a dimension name, a literal element selector, or a + /// deeper index the walk skipped). + pub(super) fn get(&self, path: &[u16]) -> Option<&'a OccurrenceSite> { + self.entries + .iter() + .find(|(p, _)| *p == path) + .map(|(_, o)| *o) + } + + /// Whether this slot recorded NO occurrences. `true` for a genuinely + /// source-free slot -- an EXCEPT default that never references `from`, whose + /// trivial-zero guard form is legitimate, or an AGG-source generator whose + /// live source (the synthetic agg) is never a recorded occurrence. A miss on + /// a NON-empty lookup, by contrast, is a walker desync -- the `missing_occurrence` + /// guard in the subscript arm keys on this so it never false-fires on a + /// legitimately source-free slot. + pub(super) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Does any occurrence of the live source (a model `Variable` or a + /// `ModuleOutput` composite) STRICTLY under `prefix` carry access shape + /// `shape`? This is the occurrence-IR form of the retired + /// `expr0_contains_live_match` lookahead: an array-reducer `App` at + /// `prefix` is frozen whole (GH #517) unless it genuinely holds the live + /// reference, which is exactly "a live-shaped occurrence lives in its + /// subtree". + /// + /// Index-nested occurrences (`other_arr[live]`) are EXCLUDED: the retired + /// `expr0_contains_live_match` only inspected subscript *heads*, never + /// recursing into a subscript's index expressions, so an index-nested + /// `live` never made the enclosing reducer "hold the live ref". The + /// occurrence IR marks those `index_nested`, so filtering on it reproduces + /// that boundary exactly (the reducer freezes whole and the bare `live` + /// stays live -- `char_reducer_index_nested_freeze`). + /// + /// A `live` source can be either a model `Variable` (the ordinary link + /// score) or a `module·port` composite (`OccurrenceRef::ModuleOutput`, the + /// `db::module_link_score_equation` live channel). The retired + /// `expr0_contains_live_match` matched on the bare-`Var` ident text and so + /// treated BOTH the same -- a composite read bare inside a reducer made the + /// reducer hold the live ref. Matching on the occurrence's resolved name + /// (whichever variant) reproduces that: a module-output live source inside a + /// reducer recurses just like a variable one, instead of freezing whole. + pub(super) fn subtree_has_live_shape( + &self, + prefix: &[u16], + live: &Ident, + shape: &RefShape, + ) -> bool { + self.entries.iter().any(|(p, o)| { + p.len() > prefix.len() + && p.starts_with(prefix) + && !o.index_nested + && &o.shape == shape + && occurrence_names_source(&o.reference, live) + }) + } +} + +/// Whether occurrence reference `reference` names the live source `live` -- +/// either a model `Variable` or a `module·port` composite +/// (`OccurrenceRef::ModuleOutput`). The wrap treats both alike (a bare read of +/// either is a live match), so the occurrence-IR predicates must too. +pub(super) fn occurrence_names_source(reference: &OccurrenceRef, live: &Ident) -> bool { + match reference { + OccurrenceRef::Variable(v) => &Ident::::new(v) == live, + OccurrenceRef::ModuleOutput { composite, .. } => { + &Ident::::new(composite) == live + } + } +} diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index 013b17138..93a6c22a8 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -10,6 +10,7 @@ use super::*; use crate::common::{CanonicalDimensionName, CanonicalElementName}; use crate::db::LtmEquation; +use crate::db::ltm_ir::OccurrenceRef; use crate::dimensions::{Dimension, NamedDimension}; fn make_named_dimension(name: &str, elements: &[&str]) -> Dimension { @@ -4756,7 +4757,8 @@ fn sgft( ) -> Result { let occ_sites = build_wrap_test_occurrences(equation_text, from, deps, source_dim_elements, iter_ctx); - let occ = OccurrenceLookup::for_slot(&occ_sites, 0); + let slot_occurrences = SlotOccurrences::new(&occ_sites); + let occ = slot_occurrences.for_slot(0); shaped_guard_form_text( equation_text, deps, @@ -4804,7 +4806,8 @@ fn wrap_missing_live_source_occurrence_is_loud_not_silent_freeze() { .into_iter() .filter(|o| !matches!(&o.reference, OccurrenceRef::Variable(v) if v == "pop")) .collect(); - let occ = OccurrenceLookup::for_slot(&desynced, 0); + let slot_occurrences = SlotOccurrences::new(&desynced); + let occ = slot_occurrences.for_slot(0); assert!( !occ.is_empty(), "the guard scope requires a non-empty lookup -- helper must survive the drop" @@ -5286,3 +5289,75 @@ fn gh526_natural_and_unthreadable_other_deps_keep_collapse() { "un-threadable dep dims keep the permissive legacy collapse; got: {partial}" ); } + +/// [`SlotOccurrences`] groups a target's occurrence stream by slot FAITHFULLY: +/// each slot's lookup holds exactly the occurrences whose `SiteId` starts with +/// that slot, with the slot prefix stripped, and a slot with no occurrences is +/// empty rather than a desync. +/// +/// This is the structural pin for the F2 refactor (the per-slot rescan was +/// quadratic in an `Ast::Arrayed` target's element count). Grouping must be +/// equivalent to the old `filter(site_id.first() == slot)` scan -- if it were +/// not, the wrap would look occurrences up in the wrong slot and freeze the +/// wrong reference, which is a silent wrong score rather than a loud failure. +/// The end-to-end equivalence is additionally pinned by the unchanged +/// `arrayed_target_slot_scores` characterization golden. +#[test] +fn slot_occurrence_index_groups_every_slot() { + use crate::db::ltm_ir::{OccurrenceRouting, OccurrenceSite, SiteId}; + + let occ_at = |path: &[u16], name: &str| OccurrenceSite { + site_id: SiteId(path.to_vec().into_boxed_slice()), + reference: OccurrenceRef::Variable(name.to_string()), + shape: RefShape::Bare, + axes: Vec::new(), + target_element: None, + routing: OccurrenceRouting::Direct, + in_reducer: false, + reducer_keys: Vec::new(), + already_lagged: false, + index_nested: false, + }; + + // Slots 0 and 2 carry occurrences; slot 1 carries none. + let stream = vec![ + occ_at(&[0, 1], "a"), + occ_at(&[0, 1, 0], "b"), + occ_at(&[2, 0], "c"), + ]; + let index = SlotOccurrences::new(&stream); + + let slot0 = index.for_slot(0); + assert!(!slot0.is_empty()); + // Paths are slot-LOCAL: `[0, 1]` is looked up as `[1]`. + assert_eq!( + slot0.get(&[1]).map(|o| match &o.reference { + OccurrenceRef::Variable(v) => v.clone(), + _ => unreachable!(), + }), + Some("a".to_string()) + ); + assert_eq!( + slot0.get(&[1, 0]).map(|o| match &o.reference { + OccurrenceRef::Variable(v) => v.clone(), + _ => unreachable!(), + }), + Some("b".to_string()) + ); + // Slot 0 must NOT see slot 2's occurrence under any path. + assert!(slot0.get(&[0]).is_none()); + + let slot2 = index.for_slot(2); + assert_eq!( + slot2.get(&[0]).map(|o| match &o.reference { + OccurrenceRef::Variable(v) => v.clone(), + _ => unreachable!(), + }), + Some("c".to_string()) + ); + + // A slot with no occurrences is legitimately EMPTY (the `missing_occurrence` + // desync guard keys on this, so it must not look like a populated slot). + assert!(index.for_slot(1).is_empty()); + assert!(index.for_slot(99).is_empty()); +} diff --git a/src/simlin-engine/src/ltm_augment_wrap_test_support.rs b/src/simlin-engine/src/ltm_augment_wrap_test_support.rs index 01884df6c..3582ce8d0 100644 --- a/src/simlin-engine/src/ltm_augment_wrap_test_support.rs +++ b/src/simlin-engine/src/ltm_augment_wrap_test_support.rs @@ -17,6 +17,7 @@ //! resolves the parent's private items. use super::*; +use crate::db::ltm_ir::OccurrenceRef; /// Reconstruct the occurrence-IR stream ([`OccurrenceSite`]s) for one target /// equation from its raw text, for the `#[cfg(test)]` wrap unit tests From d5eed63b5fba0ef6c5c068108e1d539082912576 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 17:46:49 -0700 Subject: [PATCH 11/14] engine: complete the LTM over-arity SiteId guard on both sides b0a2a6f0 guarded only the PRODUCER. Two halves of the same defect survived, both of them the silent-corruption class the guard exists to prevent. The consumer still aliased. ltm_augment::child_path built its path with `i as u16`, so a builtin child at index 65,536 wrapped to child 0's path. The IR walker deliberately records no occurrence for that child, but the wrap would then look up child 0's -- and if that sibling HAS an occurrence, the lookup succeeds with an unrelated shape, missing_occurrence never fires, and the wrap freezes the wrong reference and emits a plausible wrong score. Omitting the occurrence on one side is only safe if the other side cannot manufacture a colliding path. So the sentinel is now reserved rather than merely hoped-for: UNADDRESSABLE_CHILD is a value site_child_index NEVER returns (it costs one child index out of 65,536), and child_path maps an over-arity child to it with a checked conversion. A path containing it therefore cannot equal any recorded SiteId, so every lookup at or below such a child provably misses instead of aliasing. The App arms additionally set missing_occurrence at the overflow point, so the partial is abandoned where the problem is rather than depending on a miss further down. The producer dropped too much. The early return skipped the whole contents match, so it suppressed push_ref_site along with the occurrence -- which my commit message for b0a2a6f0 explicitly claimed it did not do. That claim was wrong. The two views are not interchangeable: the occurrence view is SiteId-keyed and must stay silent for a child it cannot address, but the per-edge view is name-keyed and feeds model_edge_shapes, where a MISSING entry is not read as "no reference" -- consumers default it to a single Bare site. So dropping the edge site would misclassify a FixedIndex/DynamicIndex reference and emit wrong element edges and link scores. Suppression is now a depth counter consulted only by push_occurrence, covering the unaddressable child's whole subtree (a nested occurrence at a truncated path would alias a sibling) while every reference keeps its real shape in the edge view. Both halves needed their own pins, and neither was covered by the producer-side boundary test: it passes whether or not child_path's conversion is checked, and the suppression counter is never raised below 65,536 arguments, so no fixture exercises it. Each is now pinned on the exact invariant -- child_path's sentinel mapping and WalkAccum's record-the-edge-anyway contract -- rather than through a 65,536-argument fixture that would strain the suite's wall-clock cap. Both are mutation-verified: restoring `as u16` fails the consumer pin, and suppressing ref sites fails the producer pin. --- src/simlin-engine/src/db/ltm_ir.rs | 54 ++++++++++-- src/simlin-engine/src/db/ltm_ir_tests.rs | 98 +++++++++++++++++++++- src/simlin-engine/src/ltm_augment.rs | 55 +++++++++--- src/simlin-engine/src/ltm_augment_tests.rs | 42 ++++++++++ 4 files changed, 226 insertions(+), 23 deletions(-) diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index ac6d8b667..2f73f5e43 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -344,6 +344,19 @@ struct WalkAccum<'a> { sites: &'a mut HashMap>, occurrences: &'a mut Vec, path: Vec, + /// `> 0` while the walk is inside a builtin child whose index a [`SiteId`] + /// element cannot address (see [`site_child_index`]). + /// + /// Only the OCCURRENCE view is suppressed there; `push_ref_site` keeps + /// running, so the per-edge view -- and therefore `model_edge_shapes` and + /// the element causal graph -- still sees every reference with its real + /// shape. Suppressing the ref site too would leave the IR with no entry for + /// the edge, and consumers default a missing entry to a single `Bare` site, + /// which would MISCLASSIFY a `FixedIndex`/`DynamicIndex` reference and emit + /// wrong element edges. It is a depth COUNTER rather than a flag because the + /// suppression must cover the unaddressable child's whole subtree: emitting + /// a nested occurrence at a truncated path would alias a sibling's `SiteId`. + suppress_occurrences: usize, } impl WalkAccum<'_> { @@ -376,6 +389,12 @@ impl WalkAccum<'_> { already_lagged: bool, index_nested: bool, ) { + // Inside an unaddressable builtin child, record NO occurrence: its path + // could not be reproduced by a consumer, and emitting it at a truncated + // path would alias a sibling. See `suppress_occurrences`. + if self.suppress_occurrences > 0 { + return; + } self.occurrences.push(RawOccurrence { site_id: SiteId(self.path.clone().into_boxed_slice()), reference, @@ -430,6 +449,7 @@ fn collect_all_reference_sites_and_occurrences( sites: &mut sites, occurrences: &mut occurrences, path: Vec::new(), + suppress_occurrences: 0, }; match ast { crate::ast::Ast::Scalar(expr) | crate::ast::Ast::ApplyToAll(_, expr) => { @@ -565,6 +585,14 @@ fn classify_occurrence_axes( .collect() } +/// The reserved [`SiteId`] path component meaning "this child is not +/// addressable". [`site_child_index`] NEVER returns it, so a path containing it +/// cannot equal any recorded occurrence's `SiteId` -- which is exactly what lets +/// the ceteris-paribus wrap append it for an over-arity child and be *certain* +/// the lookup misses instead of aliasing an unrelated sibling +/// (`ltm_augment::child_path`). +pub(crate) const UNADDRESSABLE_CHILD: u16 = u16::MAX; + /// The [`SiteId`] child index for a builtin's `n`-th content, or `None` when the /// arity exceeds what one path element can address. /// @@ -593,8 +621,12 @@ fn classify_occurrence_axes( /// occurrence IR is salsa-cached per model and every occurrence carries a boxed /// path, so widening doubles that footprint on every model to serve an arity no /// real model reaches -- and it would not remove the case, only move it. -fn site_child_index(n: usize) -> Option { - u16::try_from(n).ok() +/// +/// [`UNADDRESSABLE_CHILD`] is excluded from the addressable range so it remains a +/// value no recorded `SiteId` can contain; that reserves one child index out of +/// 65,536 and buys the consumer a provably-unmatchable path to descend under. +pub(crate) fn site_child_index(n: usize) -> Option { + u16::try_from(n).ok().filter(|&i| i != UNADDRESSABLE_CHILD) } /// `true` iff `builtin` is `PREVIOUS(...)` / `INIT(...)`: everything inside is @@ -812,11 +844,16 @@ fn walk_all_in_expr( // that child -- see its rustdoc for why that is the loud outcome. let mut child_n: usize = 0; walk_builtin_expr(builtin, |contents| { - let Some(child) = site_child_index(child_n) else { - child_n += 1; - return; - }; - acc.path.push(child); + // An unaddressable child still gets its REFERENCE SITES walked + // (edges and their shapes must stay complete); only the + // SiteId-keyed occurrence view is suppressed, for it and its + // whole subtree. + let addressable = site_child_index(child_n); + let suppressed = addressable.is_none(); + if suppressed { + acc.suppress_occurrences += 1; + } + acc.path.push(addressable.unwrap_or(UNADDRESSABLE_CHILD)); match contents { BuiltinContents::Ident(id, _) => { let canonical = Ident::::new(id); @@ -864,6 +901,9 @@ fn walk_all_in_expr( BuiltinContents::LookupTable(_) => {} } acc.path.pop(); + if suppressed { + acc.suppress_occurrences -= 1; + } child_n += 1; }); if pushed_reducer_key { diff --git a/src/simlin-engine/src/db/ltm_ir_tests.rs b/src/simlin-engine/src/db/ltm_ir_tests.rs index 7cd6c8729..f2baf8a23 100644 --- a/src/simlin-engine/src/db/ltm_ir_tests.rs +++ b/src/simlin-engine/src/db/ltm_ir_tests.rs @@ -1686,11 +1686,103 @@ fn site_child_index_declines_rather_than_wrapping() { assert_eq!(site_child_index(0), Some(0)); assert_eq!(site_child_index(1), Some(1)); - // The last addressable child. - assert_eq!(site_child_index(65_535), Some(u16::MAX)); - // One past it must DECLINE, not wrap to 0 (which would collide with the + // The last addressable child: one below `UNADDRESSABLE_CHILD`, which is + // reserved as the consumer's provably-unmatchable sentinel (see the sibling + // test `unaddressable_child_sentinel_is_never_a_real_child_index`). + assert_eq!(site_child_index(65_534), Some(65_534)); + assert_eq!(site_child_index(65_535), None); + // Past the range must DECLINE, not wrap to 0 (which would collide with the // first child's SiteId and silently mis-address the occurrence). assert_eq!(site_child_index(65_536), None); assert_eq!(site_child_index(65_537), None); assert_eq!(site_child_index(usize::MAX), None); } + +/// The unaddressable-child sentinel must be a value `site_child_index` NEVER +/// emits. That reservation is what makes the consumer's fallback provable: the +/// ceteris-paribus wrap appends `UNADDRESSABLE_CHILD` for an over-arity child, so +/// if the producer could also emit it, a recorded occurrence would share the +/// path and the lookup would alias exactly the sibling it is trying to avoid. +#[test] +fn unaddressable_child_sentinel_is_never_a_real_child_index() { + use super::{UNADDRESSABLE_CHILD, site_child_index}; + + // The last ADDRESSABLE index is one below the sentinel. + assert_eq!(site_child_index(65_534), Some(65_534)); + assert_eq!(UNADDRESSABLE_CHILD, u16::MAX); + // The sentinel's own numeric position is declined, not returned. + assert_eq!(site_child_index(UNADDRESSABLE_CHILD as usize), None); + // ...so no `n` whatsoever maps to it. + for n in [0usize, 1, 65_534, 65_535, 65_536, usize::MAX] { + assert_ne!( + site_child_index(n), + Some(UNADDRESSABLE_CHILD), + "n = {n} must not map to the reserved sentinel" + ); + } +} + +/// Occurrence suppression inside an unaddressable builtin child must NOT +/// suppress the per-EDGE reference site. +/// +/// The two views serve different consumers. The occurrence view is +/// SiteId-keyed, so a child the path cannot address must record nothing (else +/// the wrap aliases a sibling). The per-edge view is name-keyed and feeds +/// `model_edge_shapes` and the element causal graph -- and a MISSING IR entry +/// there does not mean "no reference": consumers default it to a single `Bare` +/// site, which would misclassify a `FixedIndex`/`DynamicIndex` reference and +/// emit wrong element edges and link scores. So the edge must keep its real +/// shape even when its occurrence is dropped. +/// +/// Pinned on the accumulator rather than through a 65,536-argument builtin: the +/// suppression counter is only ever raised at that arity, so a fixture-driven +/// test would be enormous while this asserts the exact invariant. +#[test] +fn suppressed_occurrences_still_record_edge_sites() { + use super::{RefShape, ReferenceSite, WalkAccum}; + use std::collections::HashMap; + + let mut sites: HashMap> = HashMap::new(); + let mut occurrences = Vec::new(); + { + let mut acc = WalkAccum { + sites: &mut sites, + occurrences: &mut occurrences, + path: vec![0], + // As if inside a builtin child whose index cannot be addressed. + suppress_occurrences: 1, + }; + acc.push_ref_site( + "pop", + RefShape::FixedIndex(vec!["nyc".to_string()]), + None, + &[], + ); + acc.push_occurrence( + super::OccurrenceRef::Variable("pop".to_string()), + RefShape::FixedIndex(vec!["nyc".to_string()]), + Vec::new(), + None, + &[], + false, + false, + ); + } + + // The occurrence is dropped (it could not be addressed)... + assert!( + occurrences.is_empty(), + "an unaddressable child must record no occurrence" + ); + // ...but the edge site survives, WITH its real shape. + let pop_sites = sites.get("pop").expect( + "the per-edge reference site must survive suppression -- a missing IR \ + entry is defaulted to `Bare` by consumers, misclassifying the reference", + ); + assert_eq!(pop_sites.len(), 1); + assert_eq!( + pop_sites[0].shape, + RefShape::FixedIndex(vec!["nyc".to_string()]), + "the edge must keep its real shape, not degrade to Bare" + ); +} diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 01542ea7b..09340f4b6 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -641,19 +641,39 @@ struct WrapCtx<'a> { occ: &'a OccurrenceLookup<'a>, } -/// Append `i` to `path`, yielding the child node's structural path. The wrap's -/// recursion mirrors `db::ltm_ir::walk_all_in_expr`'s child-index construction -/// exactly, so the path at any node equals that occurrence's `SiteId` (minus -/// the slot prefix, which [`OccurrenceLookup::for_slot`] already stripped) -- -/// the invariant `classifier_agreement_tests::assert_occurrence_stream_aligns` -/// proves corpus-wide. Cloning per descent is cheap: LTM equations are short. -fn child_path(path: &[u16], i: u16) -> Vec { +/// Append child index `i` to `path`, yielding the child node's structural path. +/// The wrap's recursion mirrors `db::ltm_ir::walk_all_in_expr`'s child-index +/// construction exactly, so the path at any node equals that occurrence's +/// `SiteId` (minus the slot prefix, which [`SlotOccurrences::for_slot`] already +/// stripped) -- the invariant +/// `classifier_agreement_tests::assert_occurrence_stream_aligns` proves +/// corpus-wide. Cloning per descent is cheap: LTM equations are short. +/// +/// `i` is a `usize` and the conversion is CHECKED, mapping an over-arity child +/// (a variadic builtin with 65,536+ arguments) to +/// [`db::ltm_ir::UNADDRESSABLE_CHILD`] rather than letting `as u16` wrap. That +/// matters because wrapping produced an EARLIER sibling's path: the lookup would +/// return an unrelated occurrence, `missing_occurrence` would never fire, and +/// the wrap would freeze the wrong reference and emit a plausible wrong score. +/// The sentinel is a value `site_child_index` never emits, so every lookup at or +/// below such a child provably MISSES -- which turns the case into the existing +/// loud skip-and-warn. Callers additionally flag it directly (see the `App` arms) +/// rather than relying only on the downstream miss. +fn child_path(path: &[u16], i: usize) -> Vec { let mut v = Vec::with_capacity(path.len() + 1); v.extend_from_slice(path); - v.push(i); + v.push(u16::try_from(i).unwrap_or(crate::db::ltm_ir::UNADDRESSABLE_CHILD)); v } +/// Whether builtin child index `i` can be addressed by a `SiteId` element. The +/// wrap's `App` arms set [`WrapOutcome::missing_occurrence`] when this is false, +/// so the partial is abandoned LOUDLY at the overflow point instead of depending +/// on a lookup miss further down. +fn child_is_addressable(i: usize) -> bool { + crate::db::ltm_ir::site_child_index(i).is_some() +} + /// The `Collapse` / `Mismatch` / `NotIterated` verdict for an iterated-dimension /// subscript on a NON-live-source dependency, derived from the occurrence IR's /// per-axis classification (`node_occ.axes`) plus the two arity facts the @@ -902,7 +922,7 @@ fn wrap_non_matching_in_previous( ctx, out, false, - &child_path(path, i as u16), + &child_path(path, i), ) } }) @@ -960,7 +980,7 @@ fn wrap_non_matching_in_previous( ctx, out, skip_index_qualification, - &child_path(path, i as u16), + &child_path(path, i), ) }) .collect(); @@ -1011,7 +1031,10 @@ fn wrap_non_matching_in_previous( if i == 0 { a } else { - wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i as u16)) + if !child_is_addressable(i) { + out.missing_occurrence = true; + } + wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i)) } }) .collect(); @@ -1054,7 +1077,13 @@ fn wrap_non_matching_in_previous( .into_iter() .enumerate() .map(|(i, a)| { - wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i as u16)) + // An over-arity child cannot be addressed by a `SiteId`, so + // every occurrence decision inside it is unavailable: abandon + // the partial loudly rather than wrap on a guessed path. + if !child_is_addressable(i) { + out.missing_occurrence = true; + } + wrap_non_matching_in_previous(a, ctx, out, &child_path(path, i)) }) .collect(); Expr0::App(UntypedBuiltinFn(name, args), loc) @@ -1798,7 +1827,7 @@ fn wrap_live_shaped_in_previous( live_shape, frozen_ref, occ, - &child_path(path, i as u16), + &child_path(path, i), ) }) .collect(); diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index 93a6c22a8..0531139af 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -5361,3 +5361,45 @@ fn slot_occurrence_index_groups_every_slot() { assert!(index.for_slot(1).is_empty()); assert!(index.for_slot(99).is_empty()); } + +/// `child_path` must map an over-arity builtin child to the reserved +/// unaddressable sentinel rather than letting `as u16` WRAP onto an earlier +/// sibling's path. +/// +/// This is the consumer half of the F3 fix, and it needs its own pin: the +/// producer-side boundary tests (`db::ltm_ir_tests`) pass whether or not this +/// conversion is checked, because they only exercise the pure index function. +/// With a wrapping cast, child 65,536 produces child 0's path, so the lookup +/// returns an UNRELATED occurrence, `missing_occurrence` never fires, and the +/// wrap freezes the wrong reference -- a plausible, silent, wrong score. +#[test] +fn child_path_maps_over_arity_child_to_the_unaddressable_sentinel() { + use crate::db::ltm_ir::UNADDRESSABLE_CHILD; + + // Ordinary children append their own index. + assert_eq!(child_path(&[3], 0), vec![3, 0]); + assert_eq!(child_path(&[3], 7), vec![3, 7]); + assert_eq!(child_path(&[], 65_534), vec![65_534]); + + // Over-arity children get the sentinel, NOT a wrapped index. `65_536 as u16` + // is 0, which is precisely the collision this must avoid. + assert_eq!(child_path(&[3], 65_536), vec![3, UNADDRESSABLE_CHILD]); + assert_ne!(child_path(&[3], 65_536), child_path(&[3], 0)); + assert_eq!(child_path(&[3], 131_072), vec![3, UNADDRESSABLE_CHILD]); + assert_ne!(child_path(&[3], 131_072), child_path(&[3], 0)); + + // `child_is_addressable` agrees with the producer's own predicate at the + // boundary, so the wrap flags exactly the children the IR omits. + assert!(child_is_addressable(0)); + assert!(child_is_addressable(65_534)); + assert!(!child_is_addressable(65_535)); + assert!(!child_is_addressable(65_536)); + + // A path containing the sentinel can never be produced by the IR walker, so + // a lookup under it provably misses rather than aliasing. + assert_eq!( + crate::db::ltm_ir::site_child_index(65_536), + None, + "the producer must decline the same child the consumer sentinels" + ); +} From 2f56c51225fd2db6a35709e0459b5a38dcf6f4c9 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 18:12:01 -0700 Subject: [PATCH 12/14] engine: pin LTM range-bound sources and the per-element doom check Three findings, two fixed and one documented. The per-element row-pinning lowering skipped subscript RANGE endpoints. rewrite_per_element_source_refs matched only IndexExpr0::Expr when recursing through another variable's subscript, so a source reference in a range bound -- other[pop[region]:3] -- passed through untouched. The IR walker DOES record an occurrence there (IndexExpr2::Range pushes children 0 and 1) and the sibling lowering substitute_reducers_in_expr0 in the same file already descends both endpoints, so this was an internal inconsistency rather than a deliberate boundary: the recorded occurrence was left un-pinned and its dimension-name subscript survived into the scalar per-element equation, which either fails to compile (a PREVIOUS-of-dim-name capture helper) or reads the wrong element. Both index branches now descend range endpoints. The partially-classifiable branch descends only where no per-axis substitution applied, since recursing after a substitution would double-pin the source's own axis. generate_per_element_link_equation did not doom on missing_occurrence. It checked only other_dep_mismatch, so a live-source occurrence the IR could not record left the source looking non-live: the wrap froze it at PREVIOUS and the score came out a clean ZERO. Every other wrap caller already dooms on it (shaped_guard_form_text, generate_scalar_to_element_equation, build_partial_equation_shaped_with_live_ref); this was the one hole through which that silent zero could still reach an emitter, so the doom contract is now uniform across all of them. module_output_ref_in_document_order cannot distinguish a SUPPRESSED module-output occurrence (one inside an unaddressable builtin child) from "the target reads no output of this module": both yield None and fall through to the signed magnitude-1 unit transfer. That is documented rather than fixed. The outcome is the same fallback an unlocatable reference has always taken -- an explicit approximation, not a plausible-looking wrong number -- and distinguishing the two means threading an unaddressable marker through the IR and this query to serve a 65,536-argument builtin, an arity far past where LTM generation is tractable at all (GH #977). The rustdoc records what the None conflates and says that if such a marker ever becomes cheap, the honest behavior is to skip the edge loudly rather than silently approximate it. The range fix is pinned by a focused test on the lowering and mutation-verified: removing the descent leaves other[pop[region]:3] verbatim and fails the pin. --- src/simlin-engine/src/db.rs | 16 ++++ src/simlin-engine/src/ltm_augment.rs | 11 ++- .../src/ltm_augment_post_transform.rs | 33 ++++++++- src/simlin-engine/src/ltm_augment_tests.rs | 74 +++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/simlin-engine/src/db.rs b/src/simlin-engine/src/db.rs index d9cb5193e..97e178a8f 100644 --- a/src/simlin-engine/src/db.rs +++ b/src/simlin-engine/src/db.rs @@ -681,6 +681,22 @@ fn module_composite_ports( /// `OccurrenceRef::ModuleOutput` occurrences in the walk's stable /// left-to-right order, so taking the first that names `module` is a /// reproducible choice over the SAME set the old scan considered. +/// +/// One case this CANNOT distinguish, deliberately: a composite read from a +/// builtin child index no `SiteId` element can address (a variadic call with +/// 65,536+ arguments) has its occurrence suppressed by `walk_all_in_expr`, so it +/// looks identical here to "`to` reads no output of `module`". Both yield `None` +/// and [`module_link_score_equation`] falls through to the signed magnitude-1 +/// `black_box_unit_transfer_equation` instead of a real ceteris-paribus partial. +/// +/// That is accepted rather than fixed. The outcome is the SAME documented +/// fallback an unlocatable reference has always taken -- an approximation, not a +/// plausible-looking wrong number -- and distinguishing the two would mean +/// threading an "unaddressable" marker through the IR and this query to serve an +/// arity no real model reaches (a 65,536-argument builtin; note the arity is also +/// far past where LTM generation is tractable at all, see GH #977). If a future +/// change makes such a marker cheap, the honest behavior is to skip the edge +/// LOUDLY here rather than silently approximate it. fn module_output_ref_in_document_order( db: &dyn Db, model: SourceModel, diff --git a/src/simlin-engine/src/ltm_augment.rs b/src/simlin-engine/src/ltm_augment.rs index 09340f4b6..4f81cdfd5 100644 --- a/src/simlin-engine/src/ltm_augment.rs +++ b/src/simlin-engine/src/ltm_augment.rs @@ -2790,6 +2790,15 @@ pub(crate) fn generate_per_element_link_equation( // full-tuple pin over-subscript a subset-dims dep), so the mismatch doom // can never fire here; the `out.other_dep_mismatch` check below is retained // defensively to preserve `wrap_changed_first_ast`'s doom contract. + // + // `out.missing_occurrence` is checked with it, and is NOT merely defensive: + // a live-source occurrence the IR could not record (a walker desync, or a + // child index a `SiteId` cannot address) leaves the source looking non-live, + // so the wrap freezes it at `PREVIOUS` and the score comes out a clean ZERO. + // Every other wrap caller already dooms on it (`shaped_guard_form_text`, + // `generate_scalar_to_element_equation`, + // `build_partial_equation_shaped_with_live_ref`); this path omitting it was + // the one hole through which that silent zero could still reach an emitter. let live_shape = RefShape::PerElement { axes: site_axes.to_vec(), }; @@ -2803,7 +2812,7 @@ pub(crate) fn generate_per_element_link_equation( Some(dims_ctx), occ, )?; - if out.other_dep_mismatch { + if out.other_dep_mismatch || out.missing_occurrence { return Err(PartialEquationError::unfreezable(to_elem_eqn_text)); } // POST-transform row-pinning lowering: rewrite the wrapped AST's live and diff --git a/src/simlin-engine/src/ltm_augment_post_transform.rs b/src/simlin-engine/src/ltm_augment_post_transform.rs index 3f14d28f1..05559cf78 100644 --- a/src/simlin-engine/src/ltm_augment_post_transform.rs +++ b/src/simlin-engine/src/ltm_augment_post_transform.rs @@ -209,7 +209,15 @@ pub(super) fn rewrite_per_element_source_refs( Expr0::Subscript(ident, indices, loc) => { if &Ident::::new(ident.as_str()) != ctx.from { // Another variable's subscript: recurse into expression - // indices (a nested source reference can hide there). + // indices (a nested source reference can hide there) AND into a + // range's two endpoints, which are also `Expr0` + // (`other[from[D, young]:3]`). The IR walker records occurrences + // under both (`IndexExpr2::Range` pushes children 0 and 1), and + // the sibling `substitute_reducers_in_expr0` descends both, so + // skipping them here left a recorded source occurrence + // un-pinned: its dimension-name subscript survived into the + // scalar equation, which either fails to compile or reads the + // wrong element. let indices = indices .into_iter() @@ -217,6 +225,12 @@ pub(super) fn rewrite_per_element_source_refs( IndexExpr0::Expr(e) => IndexExpr0::Expr( rewrite_per_element_source_refs(e, ctx, force_qualified), ), + IndexExpr0::Range(l, r, loc) => IndexExpr0::Range( + rewrite_per_element_source_refs(l, ctx, force_qualified), + rewrite_per_element_source_refs(r, ctx, force_qualified), + loc, + ), + // Wildcard / star-range / `@N` carry no `Expr0`. other => other, }) .collect(); @@ -290,7 +304,22 @@ pub(super) fn rewrite_per_element_source_refs( RawIdent::new_from_str(&part), crate::ast::Loc::default(), )), - None => idx, + // Not a resolvable iterated-dim index. A nested source + // reference can still hide inside a range endpoint + // (`from[a:from[b]]`), and the IR records it, so descend + // rather than leaving it un-pinned -- same reason as the + // other-variable branch above. An `Expr` index that did + // not substitute is left to the wrap's conservative + // handling (recursing would double-pin the source's own + // axis). + None => match idx { + IndexExpr0::Range(l, r, rloc) => IndexExpr0::Range( + rewrite_per_element_source_refs(l, ctx, force_qualified), + rewrite_per_element_source_refs(r, ctx, force_qualified), + rloc, + ), + other => other, + }, } }) .collect(); diff --git a/src/simlin-engine/src/ltm_augment_tests.rs b/src/simlin-engine/src/ltm_augment_tests.rs index 0531139af..7b3c5371e 100644 --- a/src/simlin-engine/src/ltm_augment_tests.rs +++ b/src/simlin-engine/src/ltm_augment_tests.rs @@ -5403,3 +5403,77 @@ fn child_path_maps_over_arity_child_to_the_unaddressable_sentinel() { "the producer must decline the same child the consumer sentinels" ); } + +/// The per-element row-pinning lowering must descend into a subscript RANGE's +/// endpoints, not just its plain expression indices. +/// +/// A source reference can hide in a range bound under another variable's +/// subscript (`other[pop[region]:3]`). The IR walker records an occurrence there +/// (`IndexExpr2::Range` pushes children 0 and 1) and the sibling lowering +/// `substitute_reducers_in_expr0` descends both endpoints -- but +/// `rewrite_per_element_source_refs` matched only `IndexExpr0::Expr` and passed +/// a `Range` through untouched. The recorded occurrence was then left un-pinned: +/// its dimension-name subscript survived into the scalar per-element equation, +/// which either fails to compile (a `PREVIOUS`-of-dim-name capture helper) or +/// reads the wrong element. +#[test] +fn per_element_pin_descends_into_range_endpoints() { + use crate::dimensions::DimensionsContext; + use crate::ltm_agg::AxisRead; + + let region = make_named_dimension("region", &["nyc", "boston"]); + let from_dims = vec![region.clone()]; + let source_dim_elements = vec![vec!["nyc".to_string(), "boston".to_string()]]; + let source_dim_names = vec!["region".to_string()]; + let target_iterated_dims = vec!["region".to_string()]; + let dim_ctx = DimensionsContext::from( + [datamodel::Dimension::named( + "region".to_string(), + vec!["nyc".to_string(), "boston".to_string()], + )] + .as_slice(), + ); + let iter_ctx = IteratedDimCtx { + source_dim_names: &source_dim_names, + target_iterated_dims: &target_iterated_dims, + dim_ctx: Some(&dim_ctx), + dep_dims: None, + }; + let from = Ident::::new("pop"); + let site_axes = vec![AxisRead::Iterated { + dim: "region".to_string(), + source_dim: "region".to_string(), + }]; + let row_parts_bare = vec!["boston".to_string()]; + let mut target_elem_by_dim = HashMap::new(); + target_elem_by_dim.insert("region".to_string(), ("boston".to_string(), 1usize)); + + let ctx = super::post_transform::PerElementRefCtx { + from: &from, + site_axes: &site_axes, + row_parts_bare: &row_parts_bare, + source_dim_elements: &source_dim_elements, + from_dims: &from_dims, + target_elem_by_dim: &target_elem_by_dim, + iter_ctx: &iter_ctx, + dim_ctx: &dim_ctx, + }; + + // `pop[region]` sits in the LOWER bound of a range index of `other`. + let ast = Expr0::new("other[pop[region]:3]", LexerType::Equation) + .expect("fixture parses") + .expect("fixture is non-empty"); + let lowered = super::post_transform::rewrite_per_element_source_refs(ast, &ctx, false); + let text = print_eqn(&lowered); + + assert!( + !text.contains("pop[region]"), + "the source reference inside the range bound must be pinned, not left \ + with its dimension-name subscript; got: {text}" + ); + assert!( + text.contains("boston"), + "the range-bound occurrence must be pinned to this instantiation's row; \ + got: {text}" + ); +} From 2ae6cb33eefbc09f3215adcc978d9ef6c6806996 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 18:40:47 -0700 Subject: [PATCH 13/14] engine: assert zero LTM fragment failures across all 15 char fixtures A char golden pins generated equation TEXT, so it is structurally blind to a generated equation that is STABLY unparseable: such an equation compiles to no bytecode, the variable reads a constant 0, and the golden stays green forever. Only 5 of the 15 fixtures asserted the runtime consequence, and they did it in separate test functions, so the other 10 contributed no coverage of that class. Two independent unquotable-generation bugs turned up in one day (the 1stock leading digit, and the bare-keyword class of GH #976), neither of which any text golden would have caught -- a defect class with a demonstrated recurrence rate against a check that was 1/3 deployed. The assertion now rides the shared harness. assert_char_fixture pins the golden AND the fragment-compile expectation in one call, and the expectation is a REQUIRED argument, so a new fixture cannot forget to declare it and quietly contribute nothing. Thirteen fixtures declare AllCompile. The two that legitimately produce failures declare them exactly, each with a `why`: - agg_nested_reducer: the GH #517 live-agg-inside-a-declined-outer- reducer degradation b7898692 deliberately preserves -- the partial freezes a wildcard slice, which has no LoadPrev-of-array-view path. Six entries (two target elements, each with its score plus the two PREVIOUS-capture helpers its partial synthesizes). - reducer_index_nested_freeze: this fixture's OWN model is rejected (ArrayReferenceNeedsExplicitSubscripts -- it uses a scalar as a subscript index, deliberately, to pin the Fig. 2 Q4 index-nested selection semantics at the text level), so its PREVIOUS-capture helper inherits the un-compilable subscript. Zero failures is not reachable here without changing what the fixture pins. An exact set comparison rather than a subset: a NEW failure is a silent-zero regression, and a DISAPPEARED one means the annotation went stale and should be tightened. An annotation nothing can forget to set beats a global assert with invisible carve-outs. Non-vacuity was demonstrated rather than assumed: running all fifteen as AllCompile first, exactly the two genuinely-failing fixtures failed, and loosening either annotation back to AllCompile is rejected. One honest limit, since it bears on what this buys. The guard catches a generation change that stops compiling on the shapes the fixtures cover; it does NOT catch the quoting class specifically, because no fixture carries a name the lexer cannot read bare -- verified by re-introducing the quote_ident regression, which leaves all fifteen green while the dedicated 1stock pin fails. Covering that class through the corpus would need a fixture with such a name, which is deliberately out of scope here. The dedicated pins remain the coverage for it. Also documents the producer-side asymmetry on both accumulator methods: absence in the per-edge view is defaulted to Bare by consumers (so skipping a site MISCLASSIFIES the reference), while absence in the occurrence view is safe and sometimes required. That is why suppress_occurrences gates only push_occurrence, and it is the trap worth naming so the two are not collapsed later. --- src/simlin-engine/src/db/ltm_char_tests.rs | 266 +++++++++++++++++---- src/simlin-engine/src/db/ltm_ir.rs | 21 ++ 2 files changed, 239 insertions(+), 48 deletions(-) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index c21761174..60ae115ae 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -61,21 +61,132 @@ fn render_equation(eq: &crate::db::LtmEquation) -> String { } } -/// Deterministic dump of every synthetic variable whose name matches -/// `filter`, sorted by name, one `name` line followed by its rendered -/// equation. Used as the characterization surface: the whole string is -/// pinned byte-for-byte. -fn dump_synthetic_vars(project: datamodel::Project, discovery: bool, filter: &str) -> String { +/// What a characterization fixture expects from the LTM fragment-compile +/// diagnostic pass (`model_ltm_fragment_diagnostics`). +/// +/// Every fixture must state this explicitly -- it is a required argument of +/// [`assert_char_fixture`], so a new fixture cannot forget to declare it and +/// quietly contribute no runtime coverage. +/// +/// Why the char goldens need it at all: a golden pins generated equation TEXT, +/// so it is structurally blind to a generated equation that is *stably* +/// unparseable. Such an equation compiles to no bytecode, the variable reads a +/// constant 0, and the golden stays green forever. Two independent +/// unquotable-generation bugs were found in one day (the `1stock` leading digit, +/// and the bare-keyword class of GH #976), neither of which any text golden would +/// have caught -- so the corpus asserts the runtime consequence too. +enum FragmentExpectation { + /// No LTM synthetic fragment (or implicit helper) may fail to compile. + AllCompile, + /// This fixture deliberately exercises loud warned-skip degradation. + /// `vars` is the EXACT set of variables expected to fail -- an extra + /// failure fails the test, and so does a failure that disappears (which + /// means the annotation is now stale and should be tightened). + ExpectedFailures { + /// Why these specifically cannot compile. A carve-out without a reason + /// is indistinguishable from a bug that was annotated away. + why: &'static str, + vars: &'static [&'static str], + }, +} + +/// Build the fixture's db in the configuration the whole char suite uses: +/// discovery mode ON (every fixture exercises the discovery emitters) and +/// `ltm_enabled` ON so `model_ltm_fragment_diagnostics` actually runs. +/// +/// `ltm_enabled` does not affect the dumped text -- `model_ltm_variables` is +/// called directly and does not read the flag -- it only un-gates the diagnostic +/// pass inside `model_all_diagnostics`. +fn char_fixture_db(project: &datamodel::Project) -> (SimlinDb, SourceModel, SourceProject) { use salsa::Setter; let mut db = SimlinDb::default(); let (source_project, model) = { - let sync = sync_from_datamodel(&db, &project); + let sync = sync_from_datamodel(&db, project); (sync.project, sync.models["main"].source) }; - if discovery { - source_project.set_ltm_discovery_mode(&mut db).to(true); + source_project.set_ltm_discovery_mode(&mut db).to(true); + source_project.set_ltm_enabled(&mut db).to(true); + (db, model, source_project) +} + +/// The LTM synthetic variables / implicit helpers whose fragments failed to +/// compile, sorted. This is exactly the condition +/// `model_ltm_fragment_diagnostics` reports: the variable keeps a layout slot +/// but has no bytecode, so it evaluates to a constant 0 and every score derived +/// from it is silently degraded. +fn fragment_compile_failures( + db: &SimlinDb, + model: SourceModel, + project: SourceProject, +) -> Vec { + use crate::db::{DiagnosticError, DiagnosticSeverity, collect_model_diagnostics}; + let mut failures: Vec = collect_model_diagnostics(db, model, project) + .iter() + .filter(|d| { + d.severity == DiagnosticSeverity::Warning + && matches!( + &d.error, + DiagnosticError::Assembly(msg) if msg.contains("failed to compile") + ) + }) + .map(|d| d.variable.clone().unwrap_or_default()) + .collect(); + failures.sort(); + failures +} + +/// The characterization entry point: pin the generated equation text against +/// `golden` AND assert the fixture's declared fragment-compile expectation. +/// +/// Both halves matter and neither subsumes the other. The golden catches a +/// change in generated text that still compiles; the expectation catches a +/// generated equation that stops compiling (a silent per-variable zero) without +/// changing any other fixture's text. +#[track_caller] +fn assert_char_fixture( + golden: &str, + project: datamodel::Project, + filter: &str, + expect: FragmentExpectation, +) { + let (db, model, source_project) = char_fixture_db(&project); + let actual = render_synthetic_vars(&db, model, source_project, filter); + assert_golden(golden, &actual); + + let failures = fragment_compile_failures(&db, model, source_project); + match expect { + FragmentExpectation::AllCompile => assert!( + failures.is_empty(), + "fixture `{golden}` declares AllCompile, but these LTM fragments failed to \ + compile (each keeps a layout slot with no bytecode, so it reads a constant 0 \ + and every score through it is silently degraded): {failures:?}" + ), + FragmentExpectation::ExpectedFailures { why, vars } => { + let mut expected: Vec = vars.iter().map(|v| v.to_string()).collect(); + expected.sort(); + assert_eq!( + failures, expected, + "fixture `{golden}`'s fragment-compile failures do not match its \ + declared expectation.\n declared reason: {why}\n If a NEW failure \ + appeared, that is a silent-zero regression. If a declared failure \ + DISAPPEARED, the annotation is stale -- tighten it (ideally to \ + AllCompile)." + ); + } } - let ltm = model_ltm_variables(&db, model, source_project); +} + +/// Deterministic dump of every synthetic variable whose name matches +/// `filter`, sorted by name, one `name` line followed by its rendered +/// equation. Used as the characterization surface: the whole string is +/// pinned byte-for-byte. +fn render_synthetic_vars( + db: &SimlinDb, + model: SourceModel, + source_project: SourceProject, + filter: &str, +) -> String { + let ltm = model_ltm_variables(db, model, source_project); let mut vars: Vec<&LtmSyntheticVar> = ltm .vars .iter() @@ -144,8 +255,12 @@ fn per_element_model() -> datamodel::Project { #[test] fn char_per_element_link_scores() { - let actual = dump_synthetic_vars(per_element_model(), true, "link_score\u{205A}pop"); - assert_golden("per_element_link_scores", &actual); + assert_char_fixture( + "per_element_link_scores", + per_element_model(), + "link_score\u{205A}pop", + FragmentExpectation::AllCompile, + ); } // --------------------------------------------------------------------------- @@ -189,12 +304,12 @@ fn per_element_other_deps_model() -> datamodel::Project { #[test] fn char_per_element_other_deps() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_other_deps", per_element_other_deps_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_other_deps", &actual); } // --------------------------------------------------------------------------- @@ -325,12 +440,12 @@ fn per_element_mixed_occurrences_model() -> datamodel::Project { #[test] fn char_per_element_mixed_occurrences() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_mixed_occurrences", per_element_mixed_occurrences_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_mixed_occurrences", &actual); } // --------------------------------------------------------------------------- @@ -374,12 +489,12 @@ fn per_element_mapped_occurrence_model() -> datamodel::Project { #[test] fn char_per_element_mapped_occurrence() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_mapped_occurrence", per_element_mapped_occurrence_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_mapped_occurrence", &actual); } // Finding 1 materiality guard: the mapped analogue of @@ -521,12 +636,12 @@ fn per_element_ambiguous_pin_model() -> datamodel::Project { #[test] fn char_per_element_ambiguous_pin() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_ambiguous_pin", per_element_ambiguous_pin_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_ambiguous_pin", &actual); } // Finding 1 materiality guard: a DYNAMIC feedback model whose per-element target @@ -663,12 +778,12 @@ fn per_element_index_nested_model() -> datamodel::Project { #[test] fn char_per_element_index_nested_occurrence() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_index_nested_occurrence", per_element_index_nested_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_index_nested_occurrence", &actual); } // Finding 2 materiality guard: a DYNAMIC model whose per-element target reads @@ -806,12 +921,12 @@ fn per_element_dynamic_index_model() -> datamodel::Project { #[test] fn char_per_element_dynamic_index() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "per_element_dynamic_index", per_element_dynamic_index_model(), - true, "link_score\u{205A}pop", + FragmentExpectation::AllCompile, ); - assert_golden("per_element_dynamic_index", &actual); } // Finding 1 materiality guard: the DYNAMIC-index sibling of the ambiguous-pin @@ -934,8 +1049,12 @@ fn agg_to_scalar_model() -> datamodel::Project { #[test] fn char_agg_to_scalar_target() { - let actual = dump_synthetic_vars(agg_to_scalar_model(), true, "\u{205A}agg\u{205A}0\u{2192}"); - assert_golden("agg_to_scalar_target", &actual); + assert_char_fixture( + "agg_to_scalar_target", + agg_to_scalar_model(), + "\u{205A}agg\u{205A}0\u{2192}", + FragmentExpectation::AllCompile, + ); } // --------------------------------------------------------------------------- @@ -981,8 +1100,32 @@ fn agg_nested_reducer_model() -> datamodel::Project { #[test] fn char_agg_nested_reducer() { - let actual = dump_synthetic_vars(agg_nested_reducer_model(), true, "link_score"); - assert_golden("agg_nested_reducer", &actual); + assert_char_fixture( + "agg_nested_reducer", + agg_nested_reducer_model(), + "link_score", + FragmentExpectation::ExpectedFailures { + // The GH #517 nested-live-reducer degradation this fixture exists to + // pin: the hoisted `SUM(pop[*])` held live sits inside a DECLINED + // outer reducer, so the `agg -> growth[e]` partial embeds `PREVIOUS` + // of a wildcard slice, which has no LoadPrev-of-array-view codegen + // path. It surfaces as an Assembly warning -- the LOUD warned-zero + // `b7898692` deliberately PRESERVES rather than fixes. Each of the + // two target elements contributes its score plus the two + // PREVIOUS-capture helpers its partial synthesizes. + why: "GH #517 live-agg-inside-a-declined-outer-reducer: the partial \ + freezes a wildcard slice, which cannot compile; preserved as a \ + loud warned zero, not fixed", + vars: &[ + "$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[boston]", + "$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[nyc]", + "$\u{205A}$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[boston]\u{205A}0\u{205A}arg0", + "$\u{205A}$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[boston]\u{205A}1\u{205A}arg0", + "$\u{205A}$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[nyc]\u{205A}0\u{205A}arg0", + "$\u{205A}$\u{205A}ltm\u{205A}link_score\u{205A}$\u{205A}ltm\u{205A}agg\u{205A}0\u{2192}growth[nyc]\u{205A}1\u{205A}arg0", + ], + }, + ); } // Finding 2 materiality guard: unlike every other guard in this file (which @@ -1069,8 +1212,12 @@ fn agg_to_arrayed_model() -> datamodel::Project { #[test] fn char_agg_to_arrayed_target() { - let actual = dump_synthetic_vars(agg_to_arrayed_model(), true, "\u{205A}agg\u{205A}0\u{2192}"); - assert_golden("agg_to_arrayed_target", &actual); + assert_char_fixture( + "agg_to_arrayed_target", + agg_to_arrayed_model(), + "\u{205A}agg\u{205A}0\u{2192}", + FragmentExpectation::AllCompile, + ); } // --------------------------------------------------------------------------- @@ -1094,8 +1241,12 @@ fn arrayed_agg_model() -> datamodel::Project { #[test] fn char_arrayed_agg_to_target() { - let actual = dump_synthetic_vars(arrayed_agg_model(), true, "link_score"); - assert_golden("arrayed_agg_to_target", &actual); + assert_char_fixture( + "arrayed_agg_to_target", + arrayed_agg_model(), + "link_score", + FragmentExpectation::AllCompile, + ); } // --------------------------------------------------------------------------- @@ -1126,8 +1277,12 @@ fn arrayed_target_model() -> datamodel::Project { #[test] fn char_arrayed_target_slot_scores() { - let actual = dump_synthetic_vars(arrayed_target_model(), true, "link_score\u{205A}pop"); - assert_golden("arrayed_target_slot_scores", &actual); + assert_char_fixture( + "arrayed_target_slot_scores", + arrayed_target_model(), + "link_score\u{205A}pop", + FragmentExpectation::AllCompile, + ); } // --------------------------------------------------------------------------- @@ -1178,12 +1333,27 @@ fn reducer_index_nested_model() -> datamodel::Project { #[test] fn char_reducer_index_nested_freeze() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "reducer_index_nested_freeze", reducer_index_nested_model(), - true, "link_score\u{205A}from\u{2192}to", + FragmentExpectation::ExpectedFailures { + // This fixture's OWN model does not compile: `to = SUM(w[from, *]) + + // from` uses a scalar variable as a subscript index, which the engine + // rejects with `ArrayReferenceNeedsExplicitSubscripts` (an + // Error-severity diagnostic on `to`). The fixture is deliberately + // that shape -- it pins the Fig. 2 Q4 index-nested SELECTION + // semantics at the text level -- so its `PREVIOUS`-capture helper + // inherits the un-compilable subscript. Zero failures is therefore + // not reachable here without changing what the fixture pins. + why: "the fixture model itself is rejected \ + (ArrayReferenceNeedsExplicitSubscripts: a scalar used as a \ + subscript index), so its PREVIOUS-capture helper cannot compile", + vars: &[ + "$\u{205A}$\u{205A}ltm\u{205A}link_score\u{205A}from\u{2192}to\u{205A}0\u{205A}arg0", + ], + }, ); - assert_golden("reducer_index_nested_freeze", &actual); } // Finding 2 materiality guard for the index-nested reducer-freeze shape, @@ -1280,12 +1450,12 @@ fn already_lagged_other_dep_model() -> datamodel::Project { #[test] fn char_already_lagged_other_dep() { - let actual = dump_synthetic_vars( + assert_char_fixture( + "already_lagged_other_dep", already_lagged_other_dep_model(), - true, "link_score\u{205A}from\u{2192}to", + FragmentExpectation::AllCompile, ); - assert_golden("already_lagged_other_dep", &actual); } // --------------------------------------------------------------------------- @@ -1328,12 +1498,12 @@ fn char_scalar_feeder_bare_in_hoisted_reducer() { // Every `scale -> X` score: the site-1 `scale -> total` Bare score (both // occurrences held live, changed-LAST) plus the `scale -> $⁚ltm⁚agg⁚0` // scalar-feeder score, so the whole feeder attribution is frozen. - let actual = dump_synthetic_vars( + assert_char_fixture( + "scalar_feeder_bare_in_hoisted_reducer", scalar_feeder_bare_in_hoisted_reducer_model(), - true, "link_score\u{205A}scale", + FragmentExpectation::AllCompile, ); - assert_golden("scalar_feeder_bare_in_hoisted_reducer", &actual); } // --------------------------------------------------------------------------- diff --git a/src/simlin-engine/src/db/ltm_ir.rs b/src/simlin-engine/src/db/ltm_ir.rs index 2f73f5e43..6012e1ebb 100644 --- a/src/simlin-engine/src/db/ltm_ir.rs +++ b/src/simlin-engine/src/db/ltm_ir.rs @@ -360,6 +360,18 @@ struct WalkAccum<'a> { } impl WalkAccum<'_> { + /// Record the per-EDGE view of a reference (name-keyed, feeds + /// `model_edge_shapes` and the element causal graph). + /// + /// ASYMMETRY WITH [`WalkAccum::push_occurrence`] -- do not collapse the two. + /// A missing entry means different things in the two views. Here, absence + /// does NOT mean "no reference": consumers that find no IR entry for an edge + /// fall back to a single `Bare` site, so skipping a `FixedIndex`/`DynamicIndex` + /// reference MISCLASSIFIES it and emits wrong element edges and link scores. + /// In the occurrence view, absence is safe and sometimes required -- a + /// SiteId-unaddressable child must record nothing, or the wrap's path lookup + /// aliases a sibling. That is why `suppress_occurrences` gates only + /// `push_occurrence`. fn push_ref_site( &mut self, from: &str, @@ -378,6 +390,15 @@ impl WalkAccum<'_> { }); } + /// Record the per-OCCURRENCE view of a reference (SiteId-keyed, consumed by + /// the ceteris-paribus wrap via `OccurrenceLookup`). + /// + /// ASYMMETRY WITH [`WalkAccum::push_ref_site`] -- see its docs. Absence here + /// is safe (the wrap treats a miss as "not a recorded causal reference", and + /// a live-source subscript miss additionally trips the loud + /// `missing_occurrence` guard), whereas absence in the per-edge view + /// silently misclassifies the edge as `Bare`. Suppression therefore applies + /// to this view only. #[allow(clippy::too_many_arguments)] fn push_occurrence( &mut self, From 2b72c67d9ebdcfdd9710cd53d0d88c627db3f584 Mon Sep 17 00:00:00 2001 From: Bobby Powers Date: Sun, 19 Jul 2026 19:23:41 -0700 Subject: [PATCH 14/14] engine: add an LTM char fixture whose source name cannot be bare The corpus was blind to the highest-recurrence generation bug class we have. Two independent unquotable-generation bugs turned up in one day -- the leading-digit `1stock` fixed in 17d4e7c0, and the bare-keyword class of GH #976 -- and neither was catchable by any of the fifteen existing fixtures, because a fixture only exercises the class if it CARRIES such a name. Demonstrated, not assumed: re-introducing the quote_ident regression left all fifteen green while only the dedicated 1stock unit pin failed. The corpus is the net that runs on every commit, so a class with that recurrence rate resting entirely on one unit pin -- which a future refactor could delete with nothing going red -- was the wrong place to be. This is Model A (the PerElement mixed iterated+literal shape) cloned with the source renamed `pop` -> `1pop`: an existing shape with one identifier changed, not a new model shape. `"1pop"` is a legal quoted XMILE name that canonicalizes to `1pop`, which the equation lexer cannot read bare (it only starts an identifier on XID_Start, so bare `1pop` lexes as the number 1 followed by `pop`). A leading digit and deliberately NOT a keyword: a keyword-named source would fail today against open GH #976 (`ast::needs_quoting` has no keyword check), and a fixture that is red on correct code is worse than no fixture. Both halves of the fixture's assertion are load-bearing, verified separately under the regression. The golden catches the text change (`"1pop"` -> bare `1pop`). Independently -- with the golden temporarily recaptured so the text assertion could not mask it -- the AllCompile expectation catches the runtime consequence: both per-element scores fail to compile, so each keeps a layout slot with no bytecode and reads a constant 0. That second half is the one the fifteen text-only fixtures could never have provided. The new golden file is a behavior RECORD, not a behavior change: the fifteen existing goldens are byte-identical (`git diff dea42188..HEAD -- ltm_char_golden/` is empty). --- .../unquotable_source_name.txt | 4 ++ src/simlin-engine/src/db/ltm_char_tests.rs | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 src/simlin-engine/src/db/ltm_char_golden/unquotable_source_name.txt diff --git a/src/simlin-engine/src/db/ltm_char_golden/unquotable_source_name.txt b/src/simlin-engine/src/db/ltm_char_golden/unquotable_source_name.txt new file mode 100644 index 000000000..6e8d707b0 --- /dev/null +++ b/src/simlin-engine/src/db/ltm_char_golden/unquotable_source_name.txt @@ -0,0 +1,4 @@ +$⁚ltm⁚link_score⁚1pop[boston,young]→growth[boston] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·boston] - PREVIOUS(growth[region·boston])) = 0) OR (("1pop"[region·boston,age·young] - PREVIOUS("1pop"[region·boston,age·young])) = 0) then 0 else SAFEDIV((("1pop"[boston, young] * 0.1) - PREVIOUS(growth[region·boston])), ABS((growth[region·boston] - PREVIOUS(growth[region·boston]))), 0) * SIGN(("1pop"[region·boston,age·young] - PREVIOUS("1pop"[region·boston,age·young]))) +$⁚ltm⁚link_score⁚1pop[nyc,young]→growth[nyc] +scalar: if (TIME = INITIAL_TIME) then 0 else if ((growth[region·nyc] - PREVIOUS(growth[region·nyc])) = 0) OR (("1pop"[region·nyc,age·young] - PREVIOUS("1pop"[region·nyc,age·young])) = 0) then 0 else SAFEDIV((("1pop"[nyc, young] * 0.1) - PREVIOUS(growth[region·nyc])), ABS((growth[region·nyc] - PREVIOUS(growth[region·nyc]))), 0) * SIGN(("1pop"[region·nyc,age·young] - PREVIOUS("1pop"[region·nyc,age·young]))) diff --git a/src/simlin-engine/src/db/ltm_char_tests.rs b/src/simlin-engine/src/db/ltm_char_tests.rs index 60ae115ae..875fd633f 100644 --- a/src/simlin-engine/src/db/ltm_char_tests.rs +++ b/src/simlin-engine/src/db/ltm_char_tests.rs @@ -1599,3 +1599,47 @@ fn scalar_feeder_bare_in_arrayed_reducer_compiles_and_simulates() { } } } + +// --------------------------------------------------------------------------- +// Model P: a source whose canonical name CANNOT be spelled as a bare +// identifier. +// +// A clone of Model A (the `PerElement` mixed iterated+literal shape) with the +// source renamed `pop` -> `1pop`. XMILE lets a modeler quote any name, so +// `"1pop"` is legal and canonicalizes to `1pop`; the equation lexer, though, +// only starts an identifier on `XID_Start`, so bare `1pop` lexes as the number +// `1` followed by the identifier `pop`. +// +// This fixture exists to make the CORPUS sensitive to the unquotable-generation +// class, which two independent bugs hit in one day (the `1pop`-style leading +// digit fixed in `17d4e7c0`, and the bare-keyword class of GH #976). The other +// fifteen fixtures are structurally blind to it: they pin generated TEXT, and a +// generated equation that is stably unparseable keeps its golden green forever +// while its fragment compiles to no bytecode and the score reads a constant 0. +// Verified: re-introducing the `quote_ident` regression leaves every other +// fixture green and fails THIS one, via the `AllCompile` expectation. +// +// Deliberately a leading DIGIT and not a keyword: a keyword-named source would +// fail today against open GH #976 (`ast::needs_quoting` has no keyword check), +// which is out of scope here. This fixture must be green on correct code. +// --------------------------------------------------------------------------- + +fn unquotable_source_name_model() -> datamodel::Project { + TestProject::new("unquotable_source_name_char") + .named_dimension("Region", &["nyc", "boston"]) + .named_dimension("Age", &["young", "old"]) + .array_aux("1pop[Region,Age]", "10") + .array_flow("growth[Region]", "\"1pop\"[Region, young] * 0.1", None) + .array_stock("stock[Region]", "0", &["growth"], &[], None) + .build_datamodel() +} + +#[test] +fn char_unquotable_source_name() { + assert_char_fixture( + "unquotable_source_name", + unquotable_source_name_model(), + "link_score\u{205A}1pop", + FragmentExpectation::AllCompile, + ); +}