Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/surreal_memory/engine/pipeline_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -1126,6 +1126,10 @@ async def execute(
config: BrainConfig,
) -> PipelineContext:
assert ctx.anchor_neuron is not None
# Bulk doc-training: skip emotion extraction (find_neurons type=STATE
# scans type partition; doc knowledge has no emotional valence).
if getattr(ctx, "skip_conflicts", False):
return ctx
result = self.sentiment_extractor.extract(ctx.content, language=ctx.language)

if result.valence == Valence.NEUTRAL or not result.emotion_tags:
Expand Down Expand Up @@ -1291,7 +1295,7 @@ async def execute(
storage: NeuralStorage,
config: BrainConfig,
) -> PipelineContext:
if ctx.skip_conflicts:
if getattr(ctx, "skip_conflicts", False):
return ctx

assert ctx.anchor_neuron is not None
Expand Down Expand Up @@ -1436,6 +1440,14 @@ async def execute(
config: BrainConfig,
) -> PipelineContext:
# Collect entity and concept neurons from this encode
# Bulk doc-training: skip cross-linking to existing neurons. Each
# find_neurons(type, content_exact) call scans the full type partition
# (40k+ rows on a large brain) because SDB 3.2.0's planner picks
# idx_neuron_type and filters content post-index — ~1 s/call, 3+/chunk.
# ENRICH consolidation (post-train) adds cross-links anyway.
if getattr(ctx, "skip_conflicts", False):
return ctx

linkable = [
n
for n in ctx.entity_neurons
Expand Down Expand Up @@ -1532,6 +1544,9 @@ async def execute(
) -> PipelineContext:
if ctx.anchor_neuron is None or not ctx.entity_neurons:
return ctx
# Bulk doc-training: skip (same rationale as SemanticLinkingStep).
if getattr(ctx, "skip_conflicts", False):
return ctx

anchor_id = ctx.anchor_neuron.id
new_ids = {n.id for n in ctx.neurons_created}
Expand Down
23 changes: 13 additions & 10 deletions src/surreal_memory/storage/surrealdb/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,8 +697,14 @@ async def find_neurons(
params: dict[str, Any] = {}

if type is not None:
conditions.append("type = $ntype")
params["ntype"] = type.value
# Inline type as a literal too — same planner gotcha as brain_id: a
# parameterized ``$ntype`` makes the planner pick idx_neuron_brain
# (single-col) and filter type over all brain rows, instead of the
# composite idx_neuron_type (brain_id, type). Measured 1147 ms/q vs
# single-digit ms/q with the literal on a 92k-neuron brain. type.value
# comes from the NeuronType enum (uppercase identifier), so inlining
# is injection-safe.
conditions.append(f"type = '{type.value}'")

if content_exact is not None:
conditions.append("content = $content_exact")
Expand Down Expand Up @@ -742,25 +748,22 @@ async def find_neurons_exact_batch(
) -> dict[str, Neuron]:
"""Batched exact-content lookup — one round-trip for N contents.

Overrides the base N+1 default (sequential ``find_neurons`` per content).
Uses ``content IN $contents`` with the brain_id inlined as a literal so
the planner uses the brain_id/content indexes (a parameterized brain_id
full-scans). Returns the first match per content string.
Uses ``content IN $contents`` with brain_id inlined. On a 92k-neuron
brain this is ~1.3 s/call but still 4-5x faster than the base N+1
default (each individual find_neurons with type scans the type
partition at ~750 ms; for N=8 candidates that's 6 s vs 1.3 s).
"""
if not contents:
return {}
brain_id = self._get_brain_id()
conds = [f"brain_id = {_brain_literal(brain_id)}", "content IN $contents"]
params: dict[str, Any] = {"contents": list(dict.fromkeys(contents))}
if type is not None:
conds.append("type = $ntype")
params["ntype"] = type.value
conds.append(f"type = '{type.value}'")
if ephemeral is not None:
conds.append("ephemeral = $ephemeral")
params["ephemeral"] = ephemeral
where = " AND ".join(conds)
# OMIT embedding_vec — callers only need identity/type/content, and the
# 1024-3072-float vector is ~4-8 KB/row across a result set of N matches.
rows = await self._query(f"SELECT * OMIT embedding_vec FROM neuron WHERE {where}", **params)
out: dict[str, Neuron] = {}
for r in rows:
Expand Down
Loading