From 0625c37aa30b4d8f1dc76a2bb4d5261598e5acdb Mon Sep 17 00:00:00 2001 From: kaghi Date: Sun, 7 Jun 2026 23:44:18 -0700 Subject: [PATCH 1/5] Changed the reconstruction residual from a bi-exp error to a trace-reconstruction error. Should work and converge around 8-10 iterations instead of oscillating until max iterations. --- apps/cadecon/src/lib/iteration-manager.ts | 80 ++++++++++++++++------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/apps/cadecon/src/lib/iteration-manager.ts b/apps/cadecon/src/lib/iteration-manager.ts index 6bc1ee7..d6ded4f 100644 --- a/apps/cadecon/src/lib/iteration-manager.ts +++ b/apps/cadecon/src/lib/iteration-manager.ts @@ -456,6 +456,11 @@ export async function startRun(): Promise { let bestResidual = Infinity; let bestTauR = tauR; let bestTauD = tauD; + let bestIteration = 0; + const RESIDUAL_PATIENCE = 3; // stop after this many consecutive increases + let residualIncreaseCount = 0; + const TD_STABLE_PATIENCE = 2; + let tdStableCount = 0; // Iteration 0: record initial kernel state and alpha=1 baseline batch(() => { @@ -597,6 +602,23 @@ export async function startRun(): Promise { snapshotIteration(iter + 1, tauR, tauD); }); + // Mean trace-reconstruction residual (1 - pve): This is computed over + // a fixed subset cells. We select the min-residual iteration and + // stop once the residual has risen for a few iterations (re-enabled patience + // stop, below. usually set around 3 from empircal tests). + // This replaces the biexp fit residual, which selected a later iteration and often + // saw oscillating t_d and t_r vals. There is an improvement threshold (RECON_EPS) that + // guards against noise being mistaken for improvement. The patience val guards against + // noise being mistaken for the end. + let pveSum = 0; + let pveCount = 0; + for (const sc of cellScalars.values()) { + pveSum += sc.pve; + pveCount++; + } + const meanInferencePve = pveCount > 0 ? pveSum / pveCount : 0; + const reconResidual = 1 - meanInferencePve; // minimize this + // Capture debug trace snapshot: cell 0 from first subset that has it if (rects.length > 0 && traceResults[0].size > 0) { const debugCell = rects[0].cellStart; @@ -724,39 +746,49 @@ export async function startRun(): Promise { }); }); - // Step 4: Best-residual tracking & early stop (DISABLED) - // - // TODO: The current stopping criterion uses the bi-exponential fit residual - // (||h_free - β·template||²), which measures kernel shape mismatch. This - // doesn't always work — it can be noisy or non-monotonic depending on the - // data. A more robust approach would use the trace-reconstruction residual - // (||y - α·(K*s) - b||² across cells), which directly measures how well - // the model explains the data. That's more expensive to compute but would - // be a stronger signal for when the kernel has overshot. - // - // Disabled: with damped tau updates the residual-patience early stop fires - // too aggressively before the damped parameters have had time to settle. - // Re-enable once a better stopping metric is implemented. + // Step 4: Best-iterate selection & early stop — RECONSTRUCTION RESIDUAL. // - if (medResidual < bestResidual) { - bestResidual = medResidual; + // The trace-reconstruction residual (1 - mean pve) tracks spike recovery and is + // minimized with a best-fit kernel, which the loop reaches EARLY before it + // overshoots tau_d. We therefore select the minimum-reconstruction-residual iteration + // to finalize (replacing the biexp fit residual, which selected a later, overshot + // kernel), and allow for the guard. This works by allowing the stop after RESIDUAL_PATIENCE + // consecutive iterations that dont change. Because the residual minimizes early and rises as + // tau_d overshoots, this stops shortly after the optimum. + // Overall, better kernel and same spike quality spike inference with fewer iterations!. + const RECON_EPS = 1e-3; // must beat running min by this to update the pick + if (reconResidual < bestResidual - RECON_EPS) { + bestResidual = reconResidual; bestTauR = tauR; bestTauD = tauD; + bestIteration = iter + 1; + residualIncreaseCount = 0; + } else { + residualIncreaseCount++; + if (iter > 0 && residualIncreaseCount >= RESIDUAL_PATIENCE) { + setConvergedAtIteration(iter + 1); + break; + } } - // Step 5: Convergence check (relative change in tau values) - const relChangeTauR = Math.abs(tauR - prevTauR) / (prevTauR + 1e-20); + // Secondary safety stop: tau_d stability. If the reconstruction residual + // never triggers the patience guard (e.g. a very flat recovery surface, as on + // some seeds), still stop once tau_d has plateaued so the loop doesn't run + // to maxIter unnecessarily. (tau_r jitters even after the kernel settles, + // so we key on tau_d only — the stable signal per the trajectory study.) const relChangeTauD = Math.abs(tauD - prevTauD) / (prevTauD + 1e-20); - const maxRelChange = Math.max(relChangeTauR, relChangeTauD); - if (iter > 0 && maxRelChange < convTol) { - setConvergedAtIteration(iter + 1); - break; + if (iter > 0 && relChangeTauD < convTol) { + tdStableCount++; + if (tdStableCount >= TD_STABLE_PATIENCE) { + setConvergedAtIteration(iter + 1); + break; + } + } else { + tdStableCount = 0; } } - // Use the best-residual kernel for finalization. If the loop ran to maxIter - // without early-stopping, the current tauR/tauD may have overshot. Revert to - // the iteration that produced the lowest bi-exponential fit residual. + // Use the kernel from the iteration with the minimum trace-reconstruction residual. if (bestResidual < Infinity) { tauR = bestTauR; tauD = bestTauD; From 8e4ba346a7ccdba55824b4ac7b44cbff97750853 Mon Sep 17 00:00:00 2001 From: kaghi Date: Mon, 8 Jun 2026 10:55:56 -0700 Subject: [PATCH 2/5] The trace-reconstruction residual (1 - mean pve) tracks spike recovery and is minimized with a best-fit kernel, which the loop reaches EARLY before it overshoots tau_d. We therefore select the minimum-reconstruction-residual iteration to finalize (replacing the biexp fit residual, which selected a later kernel), and allow for the guard. This works by allowing the stop after RESIDUAL_PATIENCE consecutive iterations that dont change. Because the residual minimizes early and rises as tau_d overshoots, this stops shortly after the optimum. Overall, better kernel and same spike quality spike inference with fewer iterations! Also fixed formatting issues with the iteration-manager.ts --- apps/cadecon/src/lib/iteration-manager.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/cadecon/src/lib/iteration-manager.ts b/apps/cadecon/src/lib/iteration-manager.ts index d6ded4f..b73d0fe 100644 --- a/apps/cadecon/src/lib/iteration-manager.ts +++ b/apps/cadecon/src/lib/iteration-manager.ts @@ -605,8 +605,8 @@ export async function startRun(): Promise { // Mean trace-reconstruction residual (1 - pve): This is computed over // a fixed subset cells. We select the min-residual iteration and // stop once the residual has risen for a few iterations (re-enabled patience - // stop, below. usually set around 3 from empircal tests). - // This replaces the biexp fit residual, which selected a later iteration and often + // stop, below. usually set around 3 from empircal tests). + // This replaces the biexp fit residual, which selected a later iteration and often // saw oscillating t_d and t_r vals. There is an improvement threshold (RECON_EPS) that // guards against noise being mistaken for improvement. The patience val guards against // noise being mistaken for the end. @@ -751,10 +751,10 @@ export async function startRun(): Promise { // The trace-reconstruction residual (1 - mean pve) tracks spike recovery and is // minimized with a best-fit kernel, which the loop reaches EARLY before it // overshoots tau_d. We therefore select the minimum-reconstruction-residual iteration - // to finalize (replacing the biexp fit residual, which selected a later, overshot - // kernel), and allow for the guard. This works by allowing the stop after RESIDUAL_PATIENCE + // to finalize (replacing the biexp fit residual, which selected a later + // kernel), and allow for the guard. This works by allowing the stop after RESIDUAL_PATIENCE // consecutive iterations that dont change. Because the residual minimizes early and rises as - // tau_d overshoots, this stops shortly after the optimum. + // tau_d overshoots, this stops shortly after the optimum. // Overall, better kernel and same spike quality spike inference with fewer iterations!. const RECON_EPS = 1e-3; // must beat running min by this to update the pick if (reconResidual < bestResidual - RECON_EPS) { @@ -788,7 +788,7 @@ export async function startRun(): Promise { } } - // Use the kernel from the iteration with the minimum trace-reconstruction residual. + // Use the kernel from the iteration with the minimum trace-reconstruction residual. if (bestResidual < Infinity) { tauR = bestTauR; tauD = bestTauD; From d1f301b48c6de5775239f1ecc7856476e8607e93 Mon Sep 17 00:00:00 2001 From: kaghi Date: Sun, 14 Jun 2026 23:15:02 -0700 Subject: [PATCH 3/5] Checking to see if linting error is fixed for CI testing. (previously assigned variables appeared unused for original convergence check). --- apps/cadecon/src/lib/iteration-manager.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/cadecon/src/lib/iteration-manager.ts b/apps/cadecon/src/lib/iteration-manager.ts index 12248f1..b108a83 100644 --- a/apps/cadecon/src/lib/iteration-manager.ts +++ b/apps/cadecon/src/lib/iteration-manager.ts @@ -65,6 +65,16 @@ import { reconvolveAR2 } from './reconvolve.ts'; /** Number of early free-kernel samples to skip in bi-exponential fitting. */ export const BIEXP_FIT_SKIP = 0; +/** + * DIAGNOSTIC ONLY. When true, both early-stop conditions (reconstruction-residual + * patience and tau_decay stability) are disabled, so the loop runs the full + * maxIterations. Use with a high maxIterations to capture the complete, + * untruncated residual trajectory (e.g. to observe the bi-exponential residual's + * full non-converging behavior). Best-iterate finalization still applies. + * MUST be set back to false for production runs. + */ +const DISABLE_EARLY_STOP = true; + let pool: WorkerPool | null = null; let nextJobId = 0; let pauseResolver: (() => void) | null = null; @@ -455,6 +465,7 @@ export async function startRun(): Promise { let bestResidual = Infinity; let bestTauR = tauR; let bestTauD = tauD; + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- retained for legacy best-iterate diagnostic / old selection path let bestIteration = 0; const RESIDUAL_PATIENCE = 3; // stop after this many consecutive increases let residualIncreaseCount = 0; @@ -691,6 +702,7 @@ export async function startRun(): Promise { // Step 3: Merge — median tauRise/tauDecay across subsets setRunPhase('merge'); + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- keeping for original convergence criterion (max relChange of tau_r/tau_d) const prevTauR = tauR; const prevTauD = tauD; // Extract all scalar fields in a single pass for median computation @@ -764,7 +776,7 @@ export async function startRun(): Promise { residualIncreaseCount = 0; } else { residualIncreaseCount++; - if (iter > 0 && residualIncreaseCount >= RESIDUAL_PATIENCE) { + if (!DISABLE_EARLY_STOP && iter > 0 && residualIncreaseCount >= RESIDUAL_PATIENCE) { setConvergedAtIteration(iter + 1); break; } @@ -776,7 +788,7 @@ export async function startRun(): Promise { // to maxIter unnecessarily. (tau_r jitters even after the kernel settles, // so we key on tau_d only — the stable signal per the trajectory study.) const relChangeTauD = Math.abs(tauD - prevTauD) / (prevTauD + 1e-20); - if (iter > 0 && relChangeTauD < convTol) { + if (!DISABLE_EARLY_STOP && iter > 0 && relChangeTauD < convTol) { tdStableCount++; if (tdStableCount >= TD_STABLE_PATIENCE) { setConvergedAtIteration(iter + 1); From 0d8266802199ac8fb577dafdbca7a183b71dbb42 Mon Sep 17 00:00:00 2001 From: kaghi Date: Sun, 14 Jun 2026 23:49:07 -0700 Subject: [PATCH 4/5] renamed unused variables to _bestIteration and _prevTauR. --- apps/cadecon/src/lib/iteration-manager.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/cadecon/src/lib/iteration-manager.ts b/apps/cadecon/src/lib/iteration-manager.ts index b108a83..318d994 100644 --- a/apps/cadecon/src/lib/iteration-manager.ts +++ b/apps/cadecon/src/lib/iteration-manager.ts @@ -465,8 +465,7 @@ export async function startRun(): Promise { let bestResidual = Infinity; let bestTauR = tauR; let bestTauD = tauD; - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- retained for legacy best-iterate diagnostic / old selection path - let bestIteration = 0; + let _bestIteration = 0; const RESIDUAL_PATIENCE = 3; // stop after this many consecutive increases let residualIncreaseCount = 0; const TD_STABLE_PATIENCE = 2; @@ -702,8 +701,7 @@ export async function startRun(): Promise { // Step 3: Merge — median tauRise/tauDecay across subsets setRunPhase('merge'); - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- keeping for original convergence criterion (max relChange of tau_r/tau_d) - const prevTauR = tauR; + const _prevTauR = tauR; const prevTauD = tauD; // Extract all scalar fields in a single pass for median computation const tauRises: number[] = []; @@ -772,7 +770,7 @@ export async function startRun(): Promise { bestResidual = reconResidual; bestTauR = tauR; bestTauD = tauD; - bestIteration = iter + 1; + _bestIteration = iter + 1; residualIncreaseCount = 0; } else { residualIncreaseCount++; From 588f56f71b7d3ee8c453052d2e36c15e539281bf Mon Sep 17 00:00:00 2001 From: kaghi Date: Mon, 15 Jun 2026 00:32:21 -0700 Subject: [PATCH 5/5] Removed DISABLE_EARLY_STOP flag which was added as part of a diagnostic check for the trace recon residual approach. --- apps/cadecon/src/lib/iteration-manager.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/apps/cadecon/src/lib/iteration-manager.ts b/apps/cadecon/src/lib/iteration-manager.ts index 318d994..bfe8f49 100644 --- a/apps/cadecon/src/lib/iteration-manager.ts +++ b/apps/cadecon/src/lib/iteration-manager.ts @@ -65,16 +65,6 @@ import { reconvolveAR2 } from './reconvolve.ts'; /** Number of early free-kernel samples to skip in bi-exponential fitting. */ export const BIEXP_FIT_SKIP = 0; -/** - * DIAGNOSTIC ONLY. When true, both early-stop conditions (reconstruction-residual - * patience and tau_decay stability) are disabled, so the loop runs the full - * maxIterations. Use with a high maxIterations to capture the complete, - * untruncated residual trajectory (e.g. to observe the bi-exponential residual's - * full non-converging behavior). Best-iterate finalization still applies. - * MUST be set back to false for production runs. - */ -const DISABLE_EARLY_STOP = true; - let pool: WorkerPool | null = null; let nextJobId = 0; let pauseResolver: (() => void) | null = null; @@ -774,7 +764,7 @@ export async function startRun(): Promise { residualIncreaseCount = 0; } else { residualIncreaseCount++; - if (!DISABLE_EARLY_STOP && iter > 0 && residualIncreaseCount >= RESIDUAL_PATIENCE) { + if (iter > 0 && residualIncreaseCount >= RESIDUAL_PATIENCE) { setConvergedAtIteration(iter + 1); break; } @@ -786,7 +776,7 @@ export async function startRun(): Promise { // to maxIter unnecessarily. (tau_r jitters even after the kernel settles, // so we key on tau_d only — the stable signal per the trajectory study.) const relChangeTauD = Math.abs(tauD - prevTauD) / (prevTauD + 1e-20); - if (!DISABLE_EARLY_STOP && iter > 0 && relChangeTauD < convTol) { + if (iter > 0 && relChangeTauD < convTol) { tdStableCount++; if (tdStableCount >= TD_STABLE_PATIENCE) { setConvergedAtIteration(iter + 1);