engine: compiler optimization pass (C-LEARN -33%, LTM on world3 9.8x, LTM on C-LEARN 101x)

docs/dev/rust.md

@@ -73,6 +73,22 @@ fn my_expensive_test() { ... }
 
 Prefer `--release` for expensive tests -- enumeration, simulation, and layout code can be 10-50x faster than debug.
 
+**`#[ignore]` for runtime is a judgement about today's engine, so re-take it after the engine gets faster.** An ignored gate does not run in pre-commit or CI, which means it catches nothing until someone remembers to ask for it -- `clearn_ltm_var_count_guardrail`'s own doc comment recorded a regression that slipped through for exactly that reason. After a compile-time improvement, time the ignored set and un-ignore what now fits:
+
+```bash
+cargo test -p simlin-engine --test integration --no-run
+RUSTC_BOOTSTRAP=1 cargo test -p simlin-engine --test integration -- --ignored \
+  -Z unstable-options --report-time 2>&1 | grep 'ok <' | sort -t'<' -k2 -rn
+```
+
+Then check the whole suite against CI's budget rather than trusting a developer machine, which has far more cores than a runner:
+
+```bash
+RUST_TEST_THREADS=4 taskset -c 0-3 cargo test --workspace   # approximates a CI runner
+```
+
+When a test stays ignored, say WHY in the attribute or the doc comment, and make the reason falsifiable: "runtime class" goes stale, while "executing C-LEARN under the non-JIT wasm interpreter, which no compiler speedup touches" or "a strict subset of a test that now runs by default" does not.
+
 ## Code Quality
 
 - No placeholder comments ("this is a placeholder"). Use `todo!()` or `unimplemented!()` macros for stubbed-out code, but generally continue working until the implementation is complete.

src/simlin-engine/benches/compiler.rs

@@ -9,6 +9,9 @@
 //!
 //! - `parse_mdl` -- MDL text -> `datamodel::Project` (lexing + parsing + conversion)
 //! - `bytecode_compile` -- `datamodel::Project` -> `CompiledSimulation` (bytecode generation)
+//! - `ltm_compile` -- the same, with LTM (Loops That Matter) enabled, which adds
+//!   causal-edge extraction, loop enumeration, and synthetic link/loop-score
+//!   variable generation on top of ordinary compilation
 //! - `full_pipeline` -- MDL text -> `CompiledSimulation` (all stages end-to-end)
 //! - `salsa_incremental` -- measures `compile_project_incremental` on synthetic
 //!   chain models after a single-variable equation edit or variable add/remove
@@ -29,7 +32,9 @@ use std::time::Duration;
 
 use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
 use simlin_engine::datamodel::{self, Aux, Compat, Dt, Equation, SimMethod, SimSpecs, Variable};
-use simlin_engine::db::{SimlinDb, compile_project_incremental, sync_from_datamodel_incremental};
+use simlin_engine::db::{
+    SimlinDb, compile_project_incremental, set_project_ltm_enabled, sync_from_datamodel_incremental,
+};
 use simlin_engine::open_vensim;
 
 /// Model metadata for benchmark parameterization.
@@ -132,6 +137,57 @@ fn bench_bytecode_compile(c: &mut Criterion) {
     group.finish();
 }
 
+/// Models the `ltm_compile` group runs. Deliberately not all of [`MODELS`]:
+/// C-LEARN under LTM is a multi-minute compile (~37k synthetic variables), far
+/// outside a criterion sample budget. WRLD3 is the largest model that compiles
+/// with LTM in a benchmarkable time, and it is the model this group exists for
+/// -- its 428 causal edges each get a synthetic link-score variable, so the
+/// LTM-only stages dominate its ordinary compile by roughly an order of
+/// magnitude.
+static LTM_MODELS: &[&str] = &["wrld3"];
+
+/// Benchmark: datamodel::Project -> CompiledSimulation with LTM enabled.
+///
+/// Covers the whole LTM surface -- element causal edges, loop enumeration,
+/// synthetic link/loop-score generation (`model_ltm_variables`), and the
+/// per-link fragment compiles -- on top of the ordinary bytecode compile the
+/// `bytecode_compile` group measures, so the difference between the two groups
+/// is the LTM cost.
+fn bench_ltm_compile(c: &mut Criterion) {
+    let mut group = c.benchmark_group("ltm_compile");
+    group.measurement_time(Duration::from_secs(10));
+
+    for fixture in MODELS.iter().filter(|f| LTM_MODELS.contains(&f.name)) {
+        let contents = load_model(fixture);
+        let datamodel = open_vensim(&contents).unwrap();
+
+        if !is_simulatable(&datamodel) {
+            eprintln!(
+                "skipping ltm_compile/{}: model is not simulatable",
+                fixture.name
+            );
+            continue;
+        }
+
+        group.bench_with_input(
+            BenchmarkId::from_parameter(fixture.name),
+            &datamodel,
+            |b, datamodel| {
+                b.iter(|| {
+                    let mut db = SimlinDb::default();
+                    let state = sync_from_datamodel_incremental(&mut db, datamodel, None);
+                    // The flag rides on the project input, so it must be set
+                    // before the first compile; every LTM query reads it.
+                    set_project_ltm_enabled(&mut db, state.project, true);
+                    black_box(compile_project_incremental(&db, state.project, "main").unwrap())
+                });
+            },
+        );
+    }
+
+    group.finish();
+}
+
 /// Benchmark: MDL text -> CompiledSimulation (full pipeline).
 ///
 /// Measures the entire compilation pipeline end-to-end, including parsing,
@@ -317,6 +373,7 @@ criterion_group!(
     benches,
     bench_parse_mdl,
     bench_bytecode_compile,
+    bench_ltm_compile,
     bench_full_pipeline,
     bench_incremental_equation_edit,
     bench_incremental_add_remove,

src/simlin-engine/src/ast/expr1.rs

@@ -24,14 +24,14 @@ pub enum IndexExpr1 {
 }
 
 impl IndexExpr1 {
-    pub(crate) fn from(expr: IndexExpr0) -> EquationResult<Self> {
+    pub(crate) fn from(expr: &IndexExpr0) -> EquationResult<Self> {
         let expr = match expr {
-            IndexExpr0::Wildcard(loc) => IndexExpr1::Wildcard(loc),
-            IndexExpr0::StarRange(ident, loc) => IndexExpr1::StarRange(ident.canonicalize(), loc),
+            IndexExpr0::Wildcard(loc) => IndexExpr1::Wildcard(*loc),
+            IndexExpr0::StarRange(ident, loc) => IndexExpr1::StarRange(ident.canonicalize(), *loc),
             IndexExpr0::Range(l, r, loc) => {
-                IndexExpr1::Range(Expr1::from(l)?, Expr1::from(r)?, loc)
+                IndexExpr1::Range(Expr1::from(l)?, Expr1::from(r)?, *loc)
             }
-            IndexExpr0::DimPosition(n, loc) => IndexExpr1::DimPosition(n, loc),
+            IndexExpr0::DimPosition(n, loc) => IndexExpr1::DimPosition(*n, *loc),
             IndexExpr0::Expr(e) => IndexExpr1::Expr(Expr1::from(e)?),
         };
 
@@ -72,13 +72,23 @@ pub enum Expr1 {
 }
 
 impl Expr1 {
-    pub(crate) fn from(expr: Expr0) -> EquationResult<Self> {
+    /// Lower a parsed `Expr0` into an `Expr1`, by REFERENCE.
+    ///
+    /// Deliberately not by value even though every field is either copied or
+    /// re-derived here: `Expr1` is a different tree with different identifier
+    /// types, so it is built from scratch either way, and taking `&Expr0` is
+    /// what lets `lower_ast` read a variable's parsed AST straight out of its
+    /// (shared, salsa-cached) home. Consuming it forced the caller to deep-copy
+    /// the whole `Expr0` tree -- a `Box` per node and a `String` per identifier
+    /// -- purely so this function could destroy it, which was the single
+    /// largest source of allocations in a C-LEARN compile.
+    pub(crate) fn from(expr: &Expr0) -> EquationResult<Self> {
         let expr = match expr {
-            Expr0::Const(s, n, loc) => Expr1::Const(s, n, loc),
-            Expr0::Var(id, loc) => Expr1::Var(id.canonicalize(), loc),
+            Expr0::Const(s, n, loc) => Expr1::Const(s.clone(), *n, *loc),
+            Expr0::Var(id, loc) => Expr1::Var(id.canonicalize(), *loc),
             Expr0::App(UntypedBuiltinFn(id, orig_args), loc) => {
-                let args: EquationResult<Vec<Expr1>> =
-                    orig_args.into_iter().map(Expr1::from).collect();
+                let loc = *loc;
+                let args: EquationResult<Vec<Expr1>> = orig_args.iter().map(Expr1::from).collect();
                 let mut args = args?;
 
                 macro_rules! check_arity {
@@ -301,21 +311,21 @@ impl Expr1 {
             }
             Expr0::Subscript(id, args, loc) => {
                 let args: EquationResult<Vec<IndexExpr1>> =
-                    args.into_iter().map(IndexExpr1::from).collect();
-                Expr1::Subscript(id.canonicalize(), args?, loc)
+                    args.iter().map(IndexExpr1::from).collect();
+                Expr1::Subscript(id.canonicalize(), args?, *loc)
             }
-            Expr0::Op1(op, l, loc) => Expr1::Op1(op, Box::new(Expr1::from(*l)?), loc),
+            Expr0::Op1(op, l, loc) => Expr1::Op1(*op, Box::new(Expr1::from(l)?), *loc),
             Expr0::Op2(op, l, r, loc) => Expr1::Op2(
-                op,
-                Box::new(Expr1::from(*l)?),
-                Box::new(Expr1::from(*r)?),
-                loc,
+                *op,
+                Box::new(Expr1::from(l)?),
+                Box::new(Expr1::from(r)?),
+                *loc,
             ),
             Expr0::If(cond, t, f, loc) => Expr1::If(
-                Box::new(Expr1::from(*cond)?),
-                Box::new(Expr1::from(*t)?),
-                Box::new(Expr1::from(*f)?),
-                loc,
+                Box::new(Expr1::from(cond)?),
+                Box::new(Expr1::from(t)?),
+                Box::new(Expr1::from(f)?),
+                *loc,
             ),
         };
         Ok(expr)

src/simlin-engine/src/ast/expr2.rs

@@ -86,7 +86,7 @@ impl IndexExpr2 {
         let expr = match expr {
             IndexExpr1::Wildcard(loc) => IndexExpr2::Wildcard(loc),
             IndexExpr1::StarRange(ident, loc) => {
-                IndexExpr2::StarRange(CanonicalDimensionName::from_raw(ident.as_str()), loc)
+                IndexExpr2::StarRange(CanonicalDimensionName::from(&ident), loc)
             }
             IndexExpr1::Range(l, r, loc) => {
                 IndexExpr2::Range(Expr2::from(l, ctx)?, Expr2::from(r, ctx)?, loc)
@@ -171,7 +171,7 @@ pub enum Expr2 {
 /// Provides access to variable dimension information and temp ID allocation
 pub trait Expr2Context {
     /// Get the dimensions of a variable, or None if it's a scalar
-    fn get_dimensions(&self, ident: &str) -> Option<Vec<Dimension>>;
+    fn get_dimensions(&self, ident: &str) -> Option<&[Dimension]>;
 
     /// Allocate a new temp ID for the current equation
     fn allocate_temp_id(&mut self) -> u32;
@@ -816,7 +816,7 @@ impl Expr2 {
                             && ctx.is_dimension_name(id.as_str())
                         {
                             // Convert SIZE(DimName) to a constant
-                            let dim_name = CanonicalDimensionName::from_raw(id.as_str());
+                            let dim_name = CanonicalDimensionName::from(id);
                             if let Some(len) = ctx.get_dimension_len(&dim_name) {
                                 // Return a constant expression with the dimension size
                                 return Ok(Expr2::Const(
@@ -1223,8 +1223,8 @@ mod tests {
     }
 
     impl Expr2Context for TestContext {
-        fn get_dimensions(&self, ident: &str) -> Option<Vec<Dimension>> {
-            self.dimensions.get(ident).cloned()
+        fn get_dimensions(&self, ident: &str) -> Option<&[Dimension]> {
+            self.dimensions.get(ident).map(|dims| dims.as_slice())
         }
 
         fn allocate_temp_id(&mut self) -> u32 {

src/simlin-engine/src/ast/expr3.rs

@@ -286,7 +286,7 @@ impl Expr3 {
 /// - Detecting dimension name references in subscripts
 pub trait Expr3LowerContext {
     /// Get the dimensions of a variable, or None if it's a scalar.
-    fn get_dimensions(&self, ident: &str) -> Option<Vec<Dimension>>;
+    fn get_dimensions(&self, ident: &str) -> Option<&[Dimension]>;
 
     /// Check if an identifier is a dimension name (not a variable).
     /// Used to detect A2A dimension references in subscripts.
@@ -325,7 +325,7 @@ impl IndexExpr3 {
                 // For named dimensions like Cities{Boston,NYC,LA}, this becomes StarRange("cities").
                 // The downstream compiler/evaluator must recognize that StarRange(parent_dim)
                 // means "iterate over all elements" (equivalent to IndexOp::Wildcard).
-                let dim_name = CanonicalDimensionName::from_raw(dim.name());
+                let dim_name = dim.canonical_name().clone();
                 Ok(IndexExpr3::StarRange(dim_name, *loc))
             }
             IndexExpr2::StarRange(subdim_name, loc) => {
@@ -362,13 +362,13 @@ impl IndexExpr3 {
                     && ctx.is_dimension_name(ident.as_str())
                 {
                     // Check if this is an element of the parent dimension first
-                    let element_name = CanonicalElementName::from_raw(ident.as_str());
+                    let element_name = CanonicalElementName::from(ident);
                     let is_element_of_parent = dim
                         .map(|d| d.get_offset(&element_name).is_some())
                         .unwrap_or(false);
 
                     if !is_element_of_parent {
-                        let canonical = CanonicalDimensionName::from_raw(ident.as_str());
+                        let canonical = CanonicalDimensionName::from(ident);
                         return Ok(IndexExpr3::Dimension(canonical, *loc));
                     }
                 }
@@ -401,10 +401,7 @@ impl Expr3 {
                     // which are immediately resolved to star ranges
                     let subscripts: Vec<IndexExpr3> = dims
                         .iter()
-                        .map(|dim| {
-                            let dim_name = CanonicalDimensionName::from_raw(dim.name());
-                            IndexExpr3::StarRange(dim_name, *loc)
-                        })
+                        .map(|dim| IndexExpr3::StarRange(dim.canonical_name().clone(), *loc))
                         .collect();
 
                     return Ok(Expr3::Subscript(
@@ -441,7 +438,7 @@ impl Expr3 {
                 // Validate subscript count matches dimension count.
                 // This catches cases like arr[*, *, *] on a 2D array before
                 // we hit misleading errors in individual subscript lowering.
-                if let Some(ref d) = dims
+                if let Some(d) = dims
                     && args.len() > d.len()
                 {
                     // Find the first out-of-bounds subscript for error location
@@ -450,12 +447,11 @@ impl Expr3 {
                     return eqn_err!(MismatchedDimensions, extra_loc.start, extra_loc.end);
                 }
 
-                let dims_ref = dims.as_deref();
                 let lowered_args: EquationResult<Vec<IndexExpr3>> = args
                     .iter()
                     .enumerate()
                     .map(|(i, arg)| {
-                        let dim = dims_ref.and_then(|d| d.get(i));
+                        let dim = dims.and_then(|d| d.get(i));
                         IndexExpr3::from_index_expr2(arg, dim, ctx)
                     })
                     .collect();
@@ -710,8 +706,7 @@ impl<'a> Pass1Context<'a> {
                 {
                     // Find the active dimension that matches this dimension name
                     for (dim, sub) in active_dims.iter().zip(active_subs.iter()) {
-                        let active_dim_name = CanonicalDimensionName::from_raw(dim.name());
-                        if active_dim_name.as_str() == dim_name.as_str() {
+                        if dim.name() == dim_name.as_str() {
                             // Found a match - resolve to the concrete index
                             if let Some(index) = Self::subscript_to_index(dim, sub) {
                                 let const_expr =
@@ -1529,11 +1524,12 @@ mod tests {
             .map(|e| CanonicalElementName::from_raw(e))
             .collect();
 
-        let indexed_elements: HashMap<CanonicalElementName, usize> = canonical_elements
-            .iter()
-            .enumerate()
-            .map(|(i, e)| (e.clone(), i))
-            .collect();
+        let indexed_elements: crate::common::IdentMap<CanonicalElementName, usize> =
+            canonical_elements
+                .iter()
+                .enumerate()
+                .map(|(i, e)| (e.clone(), i))
+                .collect();
 
         Dimension::Named(
             CanonicalDimensionName::from_raw(name),
@@ -1576,8 +1572,8 @@ mod tests {
     }
 
     impl Expr3LowerContext for TestLowerContext {
-        fn get_dimensions(&self, ident: &str) -> Option<Vec<Dimension>> {
-            self.dimensions.get(ident).cloned()
+        fn get_dimensions(&self, ident: &str) -> Option<&[Dimension]> {
+            self.dimensions.get(ident).map(|dims| dims.as_slice())
         }
 
         fn is_dimension_name(&self, ident: &str) -> bool {