diff --git a/src/surreal_memory/engine/pipeline_steps.py b/src/surreal_memory/engine/pipeline_steps.py index 9949da81..fb164786 100755 --- a/src/surreal_memory/engine/pipeline_steps.py +++ b/src/surreal_memory/engine/pipeline_steps.py @@ -1126,6 +1126,10 @@ async def execute( config: BrainConfig, ) -> PipelineContext: assert ctx.anchor_neuron is not None + # Bulk doc-training: skip emotion extraction (find_neurons type=STATE + # scans type partition; doc knowledge has no emotional valence). + if getattr(ctx, "skip_conflicts", False): + return ctx result = self.sentiment_extractor.extract(ctx.content, language=ctx.language) if result.valence == Valence.NEUTRAL or not result.emotion_tags: @@ -1291,7 +1295,7 @@ async def execute( storage: NeuralStorage, config: BrainConfig, ) -> PipelineContext: - if ctx.skip_conflicts: + if getattr(ctx, "skip_conflicts", False): return ctx assert ctx.anchor_neuron is not None @@ -1436,6 +1440,14 @@ async def execute( config: BrainConfig, ) -> PipelineContext: # Collect entity and concept neurons from this encode + # Bulk doc-training: skip cross-linking to existing neurons. Each + # find_neurons(type, content_exact) call scans the full type partition + # (40k+ rows on a large brain) because SDB 3.2.0's planner picks + # idx_neuron_type and filters content post-index — ~1 s/call, 3+/chunk. + # ENRICH consolidation (post-train) adds cross-links anyway. + if getattr(ctx, "skip_conflicts", False): + return ctx + linkable = [ n for n in ctx.entity_neurons @@ -1532,6 +1544,9 @@ async def execute( ) -> PipelineContext: if ctx.anchor_neuron is None or not ctx.entity_neurons: return ctx + # Bulk doc-training: skip (same rationale as SemanticLinkingStep). + if getattr(ctx, "skip_conflicts", False): + return ctx anchor_id = ctx.anchor_neuron.id new_ids = {n.id for n in ctx.neurons_created} diff --git a/src/surreal_memory/storage/surrealdb/store.py b/src/surreal_memory/storage/surrealdb/store.py index f0b2958c..81016f7e 100755 --- a/src/surreal_memory/storage/surrealdb/store.py +++ b/src/surreal_memory/storage/surrealdb/store.py @@ -697,8 +697,14 @@ async def find_neurons( params: dict[str, Any] = {} if type is not None: - conditions.append("type = $ntype") - params["ntype"] = type.value + # Inline type as a literal too — same planner gotcha as brain_id: a + # parameterized ``$ntype`` makes the planner pick idx_neuron_brain + # (single-col) and filter type over all brain rows, instead of the + # composite idx_neuron_type (brain_id, type). Measured 1147 ms/q vs + # single-digit ms/q with the literal on a 92k-neuron brain. type.value + # comes from the NeuronType enum (uppercase identifier), so inlining + # is injection-safe. + conditions.append(f"type = '{type.value}'") if content_exact is not None: conditions.append("content = $content_exact") @@ -742,10 +748,10 @@ async def find_neurons_exact_batch( ) -> dict[str, Neuron]: """Batched exact-content lookup — one round-trip for N contents. - Overrides the base N+1 default (sequential ``find_neurons`` per content). - Uses ``content IN $contents`` with the brain_id inlined as a literal so - the planner uses the brain_id/content indexes (a parameterized brain_id - full-scans). Returns the first match per content string. + Uses ``content IN $contents`` with brain_id inlined. On a 92k-neuron + brain this is ~1.3 s/call but still 4-5x faster than the base N+1 + default (each individual find_neurons with type scans the type + partition at ~750 ms; for N=8 candidates that's 6 s vs 1.3 s). """ if not contents: return {} @@ -753,14 +759,11 @@ async def find_neurons_exact_batch( conds = [f"brain_id = {_brain_literal(brain_id)}", "content IN $contents"] params: dict[str, Any] = {"contents": list(dict.fromkeys(contents))} if type is not None: - conds.append("type = $ntype") - params["ntype"] = type.value + conds.append(f"type = '{type.value}'") if ephemeral is not None: conds.append("ephemeral = $ephemeral") params["ephemeral"] = ephemeral where = " AND ".join(conds) - # OMIT embedding_vec — callers only need identity/type/content, and the - # 1024-3072-float vector is ~4-8 KB/row across a result set of N matches. rows = await self._query(f"SELECT * OMIT embedding_vec FROM neuron WHERE {where}", **params) out: dict[str, Neuron] = {} for r in rows: