From b7d039f3d1e9dd4962565b5bc29274776bd36ea0 Mon Sep 17 00:00:00 2001 From: Toni Nowak Date: Fri, 17 Jul 2026 20:49:22 +0200 Subject: [PATCH 1/2] perf(doc-trainer): skip cross-linking/emotion steps + inline type in train mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cProfile + per-query wall profiling on a 92k-neuron default brain (N=20) showed find_neurons calls at 1.1-1.4 s/q dominated: 96% DB I/O, ~1% Python. Root cause: SDB 3.2.0 planner picks idx_neuron_type and filters content post-index, scanning 40k+ rows per call. Changes (all gated on ctx.skip_conflicts = train mode): - SemanticLinkingStep + CrossMemoryLinkStep: skip (ENRICH adds cross-links post-train). Saves ~90 find_neurons calls/20 chunks. - EmotionStep: skip (doc knowledge has no emotional valence). - find_neurons: inline type as literal (enum-safe) so the planner uses idx_neuron_type composite (IndexScan vs scan+filter). - find_neurons_exact_batch: keep content IN override but inline type literal too. N+1 base default was 4-5x slower (each type+content call = 750ms). Measured on Toni's 92k default brain, N=20 chunks: 6.7 → 3.85 s/chunk (-42%). --- src/surreal_memory/engine/pipeline_steps.py | 15 ++++++++++++ src/surreal_memory/storage/surrealdb/store.py | 23 +++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/surreal_memory/engine/pipeline_steps.py b/src/surreal_memory/engine/pipeline_steps.py index 9949da81..79fba808 100755 --- a/src/surreal_memory/engine/pipeline_steps.py +++ b/src/surreal_memory/engine/pipeline_steps.py @@ -1126,6 +1126,10 @@ async def execute( config: BrainConfig, ) -> PipelineContext: assert ctx.anchor_neuron is not None + # Bulk doc-training: skip emotion extraction (find_neurons type=STATE + # scans type partition; doc knowledge has no emotional valence). + if ctx.skip_conflicts: + return ctx result = self.sentiment_extractor.extract(ctx.content, language=ctx.language) if result.valence == Valence.NEUTRAL or not result.emotion_tags: @@ -1436,6 +1440,14 @@ async def execute( config: BrainConfig, ) -> PipelineContext: # Collect entity and concept neurons from this encode + # Bulk doc-training: skip cross-linking to existing neurons. Each + # find_neurons(type, content_exact) call scans the full type partition + # (40k+ rows on a large brain) because SDB 3.2.0's planner picks + # idx_neuron_type and filters content post-index — ~1 s/call, 3+/chunk. + # ENRICH consolidation (post-train) adds cross-links anyway. + if ctx.skip_conflicts: + return ctx + linkable = [ n for n in ctx.entity_neurons @@ -1532,6 +1544,9 @@ async def execute( ) -> PipelineContext: if ctx.anchor_neuron is None or not ctx.entity_neurons: return ctx + # Bulk doc-training: skip (same rationale as SemanticLinkingStep). + if ctx.skip_conflicts: + return ctx anchor_id = ctx.anchor_neuron.id new_ids = {n.id for n in ctx.neurons_created} diff --git a/src/surreal_memory/storage/surrealdb/store.py b/src/surreal_memory/storage/surrealdb/store.py index f0b2958c..81016f7e 100755 --- a/src/surreal_memory/storage/surrealdb/store.py +++ b/src/surreal_memory/storage/surrealdb/store.py @@ -697,8 +697,14 @@ async def find_neurons( params: dict[str, Any] = {} if type is not None: - conditions.append("type = $ntype") - params["ntype"] = type.value + # Inline type as a literal too — same planner gotcha as brain_id: a + # parameterized ``$ntype`` makes the planner pick idx_neuron_brain + # (single-col) and filter type over all brain rows, instead of the + # composite idx_neuron_type (brain_id, type). Measured 1147 ms/q vs + # single-digit ms/q with the literal on a 92k-neuron brain. type.value + # comes from the NeuronType enum (uppercase identifier), so inlining + # is injection-safe. + conditions.append(f"type = '{type.value}'") if content_exact is not None: conditions.append("content = $content_exact") @@ -742,10 +748,10 @@ async def find_neurons_exact_batch( ) -> dict[str, Neuron]: """Batched exact-content lookup — one round-trip for N contents. - Overrides the base N+1 default (sequential ``find_neurons`` per content). - Uses ``content IN $contents`` with the brain_id inlined as a literal so - the planner uses the brain_id/content indexes (a parameterized brain_id - full-scans). Returns the first match per content string. + Uses ``content IN $contents`` with brain_id inlined. On a 92k-neuron + brain this is ~1.3 s/call but still 4-5x faster than the base N+1 + default (each individual find_neurons with type scans the type + partition at ~750 ms; for N=8 candidates that's 6 s vs 1.3 s). """ if not contents: return {} @@ -753,14 +759,11 @@ async def find_neurons_exact_batch( conds = [f"brain_id = {_brain_literal(brain_id)}", "content IN $contents"] params: dict[str, Any] = {"contents": list(dict.fromkeys(contents))} if type is not None: - conds.append("type = $ntype") - params["ntype"] = type.value + conds.append(f"type = '{type.value}'") if ephemeral is not None: conds.append("ephemeral = $ephemeral") params["ephemeral"] = ephemeral where = " AND ".join(conds) - # OMIT embedding_vec — callers only need identity/type/content, and the - # 1024-3072-float vector is ~4-8 KB/row across a result set of N matches. rows = await self._query(f"SELECT * OMIT embedding_vec FROM neuron WHERE {where}", **params) out: dict[str, Neuron] = {} for r in rows: From 5de2fb95321ad30ae2f1b47f3188d4d198861712 Mon Sep 17 00:00:00 2001 From: Toni Nowak Date: Fri, 17 Jul 2026 22:17:24 +0200 Subject: [PATCH 2/2] fix: use getattr for skip_conflicts (test fixtures use SimpleNamespace without it) --- src/surreal_memory/engine/pipeline_steps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/surreal_memory/engine/pipeline_steps.py b/src/surreal_memory/engine/pipeline_steps.py index 79fba808..fb164786 100755 --- a/src/surreal_memory/engine/pipeline_steps.py +++ b/src/surreal_memory/engine/pipeline_steps.py @@ -1128,7 +1128,7 @@ async def execute( assert ctx.anchor_neuron is not None # Bulk doc-training: skip emotion extraction (find_neurons type=STATE # scans type partition; doc knowledge has no emotional valence). - if ctx.skip_conflicts: + if getattr(ctx, "skip_conflicts", False): return ctx result = self.sentiment_extractor.extract(ctx.content, language=ctx.language) @@ -1295,7 +1295,7 @@ async def execute( storage: NeuralStorage, config: BrainConfig, ) -> PipelineContext: - if ctx.skip_conflicts: + if getattr(ctx, "skip_conflicts", False): return ctx assert ctx.anchor_neuron is not None @@ -1445,7 +1445,7 @@ async def execute( # (40k+ rows on a large brain) because SDB 3.2.0's planner picks # idx_neuron_type and filters content post-index — ~1 s/call, 3+/chunk. # ENRICH consolidation (post-train) adds cross-links anyway. - if ctx.skip_conflicts: + if getattr(ctx, "skip_conflicts", False): return ctx linkable = [ @@ -1545,7 +1545,7 @@ async def execute( if ctx.anchor_neuron is None or not ctx.entity_neurons: return ctx # Bulk doc-training: skip (same rationale as SemanticLinkingStep). - if ctx.skip_conflicts: + if getattr(ctx, "skip_conflicts", False): return ctx anchor_id = ctx.anchor_neuron.id