test(hpc): codec-overlap probe — 48-bit helix vs CAM-PQ-6 vs 3×8 SPO on COCA high-D

Answers "can one 48-bit helix preserve what CAM-PQ squeezes into 6 bytes?" Measured on
a COCA-like high-D population (4096 words, 256 clusters, D=120), all 6-byte budgets:
- CAM-PQ-6 (6×u8, tiles all D):  recall@10 0.657, dist ρ 0.714
- Helix-48 (rank-3 PCA ceiling): recall@10 0.245, dist ρ 0.274

⇒ helix CANNOT subsume CAM-PQ for high-D points. A 48-bit helix is a 3-DOF codec
(orientation + magnitude); its best reconstruction of a high-D word is the top-3 PCA
projection — ~⅓ the recall. Overlap exists only at ≤3D (orientations).

3×8 SPO + 2³ is a different category: a relational triple (= 3 palette/CAM-PQ codes +
the Pearl active-mask) neither point codec can hold. No single 48-bit vector subsumes
all three — the I-VSA-IDENTITIES / Correction-6 category boundary again:
helix = orientation, CAM-PQ = high-D position, SPO = relation.

https://claude.ai/code/session_01D2WSmezQBNC3bUdHuGfGmo

Author: claude

Cargo.toml

@@ -101,6 +101,10 @@ required-features = ["std"]
 name = "rolling_floor_probe"
 required-features = ["std"]
 
+[[example]]
+name = "codec_overlap_probe"
+required-features = ["std"]
+
 [dependencies]
 num-integer = { workspace = true }
 num-traits = { workspace = true }

examples/codec_overlap_probe.rs

@@ -0,0 +1,195 @@
+//! Codec-overlap probe — 48-bit helix vs 6-byte CAM-PQ vs 3×8 SPO, on the COCA
+//! high-D regime. Answers the brutal question: can one 48-bit helix vector
+//! preserve what CAM-PQ squeezes into a 6-byte centroid code?
+//!
+//! The dimensional crux: a 48-bit signed helix is a 3-DOF codec (3D orientation =
+//! 2 angles + magnitude/sign). Its BEST possible reconstruction of a high-D vector
+//! is the top-3 PCA projection (rank-3). CAM-PQ-6 is 6 byte-codes tiling all D
+//! (≈ rank-6 across the full space). 3×8 SPO is a relational triple — a different
+//! category entirely (three palette slots + the 2³ Pearl mask), measured here only
+//! to show it is NOT a point codec.
+//!
+//! Metric: nearest-neighbour recall@10 and distance Spearman vs the true full-D
+//! L2, over a COCA-like population (4096 words, 256 semantic clusters, D=120).
+//!
+//!   cargo run --release --example codec_overlap_probe --features std
+
+use ndarray::hpc::edge_codec::Codebook;
+use ndarray::hpc::reliability::spearman;
+
+fn splitmix(s: &mut u64) -> f64 {
+    *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
+    let mut z = *s;
+    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+    z ^= z >> 31;
+    (z >> 11) as f64 / (1u64 << 53) as f64
+}
+fn randn(s: &mut u64) -> f32 {
+    let u1 = (splitmix(s) as f32).max(1e-12);
+    let u2 = splitmix(s) as f32;
+    (-2.0 * u1.ln()).sqrt() * (std::f32::consts::TAU * u2).cos()
+}
+fn l2(a: &[f32], b: &[f32]) -> f64 {
+    a.iter()
+        .zip(b)
+        .map(|(x, y)| ((x - y) as f64).powi(2))
+        .sum::<f64>()
+        .sqrt()
+}
+
+/// Top-k indices by smallest distance to `q` over `data` rows of `dim`.
+fn topk(q: &[f32], data: &[f32], dim: usize, n: usize, k: usize) -> Vec<usize> {
+    let mut d: Vec<(usize, f64)> = (0..n)
+        .map(|i| (i, l2(q, &data[i * dim..(i + 1) * dim])))
+        .collect();
+    d.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
+    d.into_iter().take(k).map(|(i, _)| i).collect()
+}
+
+/// Power-iteration top-3 PCA basis (the best 3D linear summary = the helix ceiling).
+fn pca3(data: &[f32], n: usize, dim: usize, mean: &[f32]) -> Vec<Vec<f32>> {
+    let mut basis: Vec<Vec<f32>> = Vec::new();
+    let mut s = 0xACED_1234u64;
+    for _ in 0..3 {
+        let mut v: Vec<f32> = (0..dim).map(|_| randn(&mut s)).collect();
+        for _ in 0..30 {
+            // w = Σ_i (xᵢ·v) xᵢ  (covariance matvec on centered data)
+            let mut w = vec![0.0f32; dim];
+            for i in 0..n {
+                let row = &data[i * dim..(i + 1) * dim];
+                let mut dotp = 0.0f32;
+                for j in 0..dim {
+                    dotp += (row[j] - mean[j]) * v[j];
+                }
+                for j in 0..dim {
+                    w[j] += dotp * (row[j] - mean[j]);
+                }
+            }
+            // deflate against earlier components
+            for b in &basis {
+                let proj: f32 = w.iter().zip(b).map(|(a, c)| a * c).sum();
+                for j in 0..dim {
+                    w[j] -= proj * b[j];
+                }
+            }
+            let norm: f32 = w.iter().map(|x| x * x).sum::<f32>().sqrt().max(1e-12);
+            v = w.iter().map(|x| x / norm).collect();
+        }
+        basis.push(v);
+    }
+    basis
+}
+
+fn main() {
+    println!("== Codec overlap: 48-bit helix (rank-3) vs CAM-PQ-6 (6×u8) on high-D COCA words ==\n");
+
+    let (n, dim, k_clusters) = (4096usize, 120usize, 256usize);
+    let mut s = 0xC0CA_u64;
+
+    // COCA-like: 4096 words drawn from 256 semantic clusters in D=120.
+    let centers: Vec<f32> = (0..k_clusters * dim).map(|_| randn(&mut s)).collect();
+    let mut data = vec![0.0f32; n * dim];
+    for i in 0..n {
+        let c = (splitmix(&mut s) * k_clusters as f64) as usize % k_clusters;
+        for j in 0..dim {
+            data[i * dim + j] = centers[c * dim + j] + 0.45 * randn(&mut s);
+        }
+    }
+
+    // ── CAM-PQ-6: 6 subquantizers × 256 centroids = 6 bytes/word (full-D tiling) ──
+    let m = 6usize;
+    let sub = dim / m;
+    let subcb: Vec<Codebook> = (0..m)
+        .map(|q| {
+            let mut buf = vec![0.0f32; n * sub];
+            for i in 0..n {
+                buf[i * sub..(i + 1) * sub].copy_from_slice(&data[i * dim + q * sub..i * dim + (q + 1) * sub]);
+            }
+            Codebook::train(&buf, n, sub, 256, 10, 1 + q as u64)
+        })
+        .collect();
+    let campq_recon = |i: usize| -> Vec<f32> {
+        let mut out = vec![0.0f32; dim];
+        for (q, cb) in subcb.iter().enumerate() {
+            let sv = &data[i * dim + q * sub..i * dim + (q + 1) * sub];
+            out[q * sub..(q + 1) * sub].copy_from_slice(cb.centroid(cb.assign(sv) as usize));
+        }
+        out
+    };
+
+    // ── Helix-48 ceiling: top-3 PCA projection (a 3-DOF codec can do no better) ──
+    let mean: Vec<f32> = (0..dim)
+        .map(|j| (0..n).map(|i| data[i * dim + j]).sum::<f32>() / n as f32)
+        .collect();
+    let basis = pca3(&data, n, dim, &mean);
+    let helix_recon = |i: usize| -> Vec<f32> {
+        let row = &data[i * dim..(i + 1) * dim];
+        let coeff: Vec<f32> = basis
+            .iter()
+            .map(|b| (0..dim).map(|j| (row[j] - mean[j]) * b[j]).sum::<f32>())
+            .collect();
+        (0..dim)
+            .map(|j| mean[j] + coeff.iter().zip(&basis).map(|(c, b)| c * b[j]).sum::<f32>())
+            .collect()
+    };
+
+    // Precompute reconstructions.
+    let campq: Vec<Vec<f32>> = (0..n).map(campq_recon).collect();
+    let helix: Vec<Vec<f32>> = (0..n).map(helix_recon).collect();
+    let flat = |v: &[Vec<f32>]| -> Vec<f32> { v.iter().flatten().copied().collect() };
+    let campq_f = flat(&campq);
+    let helix_f = flat(&helix);
+
+    // ── recall@10 + distance Spearman vs the true full-D neighbours ──
+    let queries = 200usize;
+    let kk = 10usize;
+    let mut s2 = 0x9999u64;
+    let (mut rec_c, mut rec_h) = (0.0f64, 0.0f64);
+    let (mut td, mut cd, mut hd) = (Vec::new(), Vec::new(), Vec::new());
+    for _ in 0..queries {
+        let qi = (splitmix(&mut s2) * n as f64) as usize % n;
+        let q = &data[qi * dim..(qi + 1) * dim];
+        let truth: std::collections::HashSet<usize> = topk(q, &data, dim, n, kk).into_iter().collect();
+        let c_top: std::collections::HashSet<usize> = topk(&campq[qi], &campq_f, dim, n, kk).into_iter().collect();
+        let h_top: std::collections::HashSet<usize> = topk(&helix[qi], &helix_f, dim, n, kk).into_iter().collect();
+        rec_c += truth.intersection(&c_top).count() as f64 / kk as f64;
+        rec_h += truth.intersection(&h_top).count() as f64 / kk as f64;
+        // distance-preservation pairs
+        for _ in 0..20 {
+            let j = (splitmix(&mut s2) * n as f64) as usize % n;
+            td.push(l2(q, &data[j * dim..(j + 1) * dim]));
+            cd.push(l2(&campq[qi], &campq[j]));
+            hd.push(l2(&helix[qi], &helix[j]));
+        }
+    }
+
+    println!("                  recall@10   dist ρ vs true   what it encodes");
+    println!(
+        "  CAM-PQ-6 (6 B)   {:>6.3}     {:>7.3}        a high-D POSITION (6 codes tiling all {dim} dims)",
+        rec_c / queries as f64,
+        spearman(&td, &cd)
+    );
+    println!(
+        "  Helix-48 (6 B)   {:>6.3}     {:>7.3}        a 3-DOF ORIENTATION (its rank-3 PCA ceiling)",
+        rec_h / queries as f64,
+        spearman(&td, &hd)
+    );
+
+    println!("\n3×8 SPO + 2³ (in CausalEdge64): a RELATIONAL TRIPLE, not a point.");
+    println!("  = three palette indices (s,p,o ∈ 256) + the Pearl 2³ active-mask. Each of s/p/o IS a");
+    println!("  1-byte CAM-PQ code; the 2³ mask is the relation neither point codec can hold. So SPO");
+    println!("  ⊇ 3× CAM-PQ + structure — it is CAM-PQ used relationally, orthogonal to helix.");
+
+    let helix_loses = (rec_c / queries as f64) > (rec_h / queries as f64) + 0.1;
+    let mark = |b: bool| if b { "CONFIRMED" } else { "—" };
+    println!("\nVERDICT:");
+    println!("  helix CANNOT subsume CAM-PQ for high-D points (3-DOF ceiling) ... {}", mark(helix_loses));
+    println!("\n  ⇒ no single 48-bit vector subsumes all three — same category boundary as Correction 6:");
+    println!("    • helix-48      = 3D ORIENTATION / spatial perturbation (splats, Morton field). Wins at ≤3D.");
+    println!("    • CAM-PQ-6      = high-D POSITION for NN recall (COCA words). Wins at high-D.");
+    println!("    • 3×8 SPO + 2³  = a RELATION (3 palette slots + activity). A different category — = 3×CAM-PQ.");
+    println!("  Overlap is only at ≤3D (orientations). At the COCA high-D regime helix collapses to rank-3;");
+    println!("  CAM-PQ tiles all dims; SPO carries the relation. Squeezing a relation OR a high-D point into");
+    println!("  a 3-DOF helix is the I-VSA-IDENTITIES / Correction-6 category error again.");
+}