perf(vectors): bump cocoindex 1.0.7→1.0.16; fix mount_table_target API

pyproject.toml

@@ -32,17 +32,17 @@ classifiers = [
 # lexical (keyword) backend over the symbol graph (search_lexical.py); every other platform
 # is unchanged.
 dependencies = [
-    "cocoindex[lancedb]>=1.0.7,<2; sys_platform != 'darwin' or platform_machine != 'x86_64'",
+    "cocoindex[lancedb]>=1.0.15,<2; sys_platform != 'darwin' or platform_machine != 'x86_64'",
     "ladybug>=0.17.1,<0.18",
-    "lancedb>=0.25.3,<0.31; sys_platform != 'darwin' or platform_machine != 'x86_64'",
+    "lancedb>=0.34,<0.36; sys_platform != 'darwin' or platform_machine != 'x86_64'",
     "mcp>=1.27.0,<2",
     "numpy>=1.26.4,<2.5",
     "pathspec>=1.0.4,<2",
-    "pyarrow>=23.0.1,<24",
+    "pyarrow>=23.0.1,<26",
     "pydantic>=2.0,<3",
     "PyYAML>=6.0.3,<7",
     "questionary>=2.0,<3",
-    "rich>=14,<15",
+    "rich>=14,<16",
     "sentence-transformers>=5.4.0,<6; sys_platform != 'darwin' or platform_machine != 'x86_64'",
     "tree-sitter>=0.25.2,<0.26",
     "tree-sitter-java>=0.23.5,<0.24",

src/java_codebase_rag/index/java_index_flow_lancedb.py

@@ -84,19 +84,16 @@
 
 splitter = RecursiveSplitter()
 
-# cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB Rewrite/compaction
-# transaction) as a *background* asyncio task after every
-# ``num_transactions_before_optimize`` mutation batches (default 50). That
-# background Rewrite races the concurrent ``table.delete()`` (Delete)
-# transactions emitted by later batches, and LanceDB does not allow a Rewrite
-# to commit concurrently with a Delete (upstream lancedb#1504), which floods
-# stderr with "Retryable commit conflict ... preempted by concurrent
-# transaction Delete". Setting this effectively to infinity disables the
-# in-flight background optimize; the serialized post-flow optimize in
-# ``lance_optimize.optimize_lance_tables`` then compacts the table with no
-# concurrent writers. ``optimize()`` is pure maintenance (compact/prune/index);
-# upsert/delete correctness via merge_insert does not depend on it.
-_NUM_TXN_BEFORE_OPTIMIZE = 10**12
+# LanceDB table optimization: cocoindex >=1.0.15 runs ``table.optimize()``
+# INLINE during merge_insert commits, gated by stats (only when small fragments
+# accumulate — see _RowHandler._maybe_optimize / _evaluate_optimize). That
+# replaces the old 1.0.7 *background* asyncio optimize that raced concurrent
+# Deletes (lancedb#1504 commit conflicts) and which we used to disable via
+# ``num_transactions_before_optimize`` (kwarg removed in 1.0.16). Being inline,
+# it no longer races anything. ``lance_optimize.optimize_lance_tables`` still
+# runs a final serialized compaction post-flow. ``optimize()`` is pure
+# maintenance (compact/prune/index); upsert/delete correctness via merge_insert
+# does not depend on it.
 
 
 # --- Vectors-phase progress emission (JCIRAG_PROGRESS kind=vectors) -----------
@@ -618,7 +615,6 @@ async def app_main() -> None:
         LANCE_DB,
         LANCE_TABLE_NAMES[0],
         java_schema,
-        num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
     )
 
     sql_schema = await lancedb.TableSchema.from_class(
@@ -629,7 +625,6 @@ async def app_main() -> None:
         LANCE_DB,
         LANCE_TABLE_NAMES[1],
         sql_schema,
-        num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
     )
 
     yaml_schema = await lancedb.TableSchema.from_class(
@@ -640,7 +635,6 @@ async def app_main() -> None:
         LANCE_DB,
         LANCE_TABLE_NAMES[2],
         yaml_schema,
-        num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE,
     )
 
     project_root = coco.use_context(PROJECT_ROOT)

src/java_codebase_rag/lance_optimize.py

@@ -1,21 +1,20 @@
 """Serialized post-flow LanceDB optimize with commit-conflict retry.
 
-cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB **Rewrite**/compaction
-transaction) as a *background* ``asyncio`` task that races concurrent
-``table.delete()`` (**Delete**) transactions emitted by later mutation batches.
-LanceDB does not allow a Rewrite to commit concurrently with a Delete
-(upstream lancedb#1504 — "We do not support concurrent deletes right now"),
-which surfaces as a flood of::
+Historically (cocoindex 1.0.7) this existed because cocoindex scheduled
+``table.optimize()`` (a LanceDB **Rewrite**/compaction) as a *background*
+``asyncio`` task that raced concurrent ``table.delete()`` (**Delete**)
+transactions — LanceDB does not allow a Rewrite to commit concurrently with a
+Delete (upstream lancedb#1504), surfacing as a flood of::
 
     RuntimeError: lance error: Retryable commit conflict for version N: \
 This Rewrite transaction was preempted by concurrent transaction Delete ...
 
-To eliminate the race, the flow (``java_index_flow_lancedb.py``) disables the
-in-flight background optimize entirely by raising
-``num_transactions_before_optimize`` to a value that is effectively never
-reached. This module then performs a *single*, serialized optimize after the
-flow returns (exit 0 → no concurrent writers), retrying the rare residual
-commit conflict that two internal compaction passes can still produce.
+cocoindex >=1.0.15 made optimize **inline and stats-driven** (only compacts
+when small fragments accumulate, inside the merge_insert commit path), so the
+race is gone and the flow no longer disables anything. This module still runs a
+*single*, serialized optimize after the flow returns (exit 0 → no concurrent
+writers) as a clean final compaction + scalar/FTS index build, retrying the rare
+residual commit conflict that two internal compaction passes can still produce.
 """
 from __future__ import annotations