From 0e789d88341fa3a55d056aaf152365012ec5bfdc Mon Sep 17 00:00:00 2001 From: Dmitry Teryaev Date: Sat, 11 Jul 2026 00:44:27 +0300 Subject: [PATCH] =?UTF-8?q?perf(vectors):=20bump=20cocoindex=201.0.7?= =?UTF-8?q?=E2=86=921.0.16;=20fix=20mount=5Ftable=5Ftarget=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump the vector trio to stay current and close an install-drift: a fresh `pip install` resolved to cocoindex 1.0.11 (our lancedb<0.31 cap blocks 1.0.15+, which requires lancedb>=0.34), so dev/CI/new users sat on different versions. Pin cocoindex>=1.0.15 (+ lancedb>=0.34, pyarrow<26, rich<16). cocoindex 1.0.16 removed lancedb.mount_table_target(num_transactions_before_optimize=...): optimize is now inline + stats-driven (_RowHandler._maybe_optimize), so the old background-optimize / Deletes race (lancedb#1504) is gone. Drop the kwarg + the _NUM_TXN_BEFORE_OPTIMIZE disable constant; lance_optimize.py still runs a final serialized compaction. Measured on Shopizer (1210 files / 3475 chunks): bulk reprocess vectors 27.5s→23.5s (~15%). No-change / 1-file increments unchanged. The single- component loop is retained — mount_each was benchmarked and rejected (#380, closed wontfix). Co-Authored-By: Claude --- pyproject.toml | 8 +++--- .../index/java_index_flow_lancedb.py | 26 +++++++------------ src/java_codebase_rag/lance_optimize.py | 23 ++++++++-------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a392c34..b36f9ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,17 +32,17 @@ classifiers = [ # lexical (keyword) backend over the symbol graph (search_lexical.py); every other platform # is unchanged. dependencies = [ - "cocoindex[lancedb]>=1.0.7,<2; sys_platform != 'darwin' or platform_machine != 'x86_64'", + "cocoindex[lancedb]>=1.0.15,<2; sys_platform != 'darwin' or platform_machine != 'x86_64'", "ladybug>=0.17.1,<0.18", - "lancedb>=0.25.3,<0.31; sys_platform != 'darwin' or platform_machine != 'x86_64'", + "lancedb>=0.34,<0.36; sys_platform != 'darwin' or platform_machine != 'x86_64'", "mcp>=1.27.0,<2", "numpy>=1.26.4,<2.5", "pathspec>=1.0.4,<2", - "pyarrow>=23.0.1,<24", + "pyarrow>=23.0.1,<26", "pydantic>=2.0,<3", "PyYAML>=6.0.3,<7", "questionary>=2.0,<3", - "rich>=14,<15", + "rich>=14,<16", "sentence-transformers>=5.4.0,<6; sys_platform != 'darwin' or platform_machine != 'x86_64'", "tree-sitter>=0.25.2,<0.26", "tree-sitter-java>=0.23.5,<0.24", diff --git a/src/java_codebase_rag/index/java_index_flow_lancedb.py b/src/java_codebase_rag/index/java_index_flow_lancedb.py index aa7160e..6db439f 100644 --- a/src/java_codebase_rag/index/java_index_flow_lancedb.py +++ b/src/java_codebase_rag/index/java_index_flow_lancedb.py @@ -84,19 +84,16 @@ splitter = RecursiveSplitter() -# cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB Rewrite/compaction -# transaction) as a *background* asyncio task after every -# ``num_transactions_before_optimize`` mutation batches (default 50). That -# background Rewrite races the concurrent ``table.delete()`` (Delete) -# transactions emitted by later batches, and LanceDB does not allow a Rewrite -# to commit concurrently with a Delete (upstream lancedb#1504), which floods -# stderr with "Retryable commit conflict ... preempted by concurrent -# transaction Delete". Setting this effectively to infinity disables the -# in-flight background optimize; the serialized post-flow optimize in -# ``lance_optimize.optimize_lance_tables`` then compacts the table with no -# concurrent writers. ``optimize()`` is pure maintenance (compact/prune/index); -# upsert/delete correctness via merge_insert does not depend on it. -_NUM_TXN_BEFORE_OPTIMIZE = 10**12 +# LanceDB table optimization: cocoindex >=1.0.15 runs ``table.optimize()`` +# INLINE during merge_insert commits, gated by stats (only when small fragments +# accumulate — see _RowHandler._maybe_optimize / _evaluate_optimize). That +# replaces the old 1.0.7 *background* asyncio optimize that raced concurrent +# Deletes (lancedb#1504 commit conflicts) and which we used to disable via +# ``num_transactions_before_optimize`` (kwarg removed in 1.0.16). Being inline, +# it no longer races anything. ``lance_optimize.optimize_lance_tables`` still +# runs a final serialized compaction post-flow. ``optimize()`` is pure +# maintenance (compact/prune/index); upsert/delete correctness via merge_insert +# does not depend on it. # --- Vectors-phase progress emission (JCIRAG_PROGRESS kind=vectors) ----------- @@ -618,7 +615,6 @@ async def app_main() -> None: LANCE_DB, LANCE_TABLE_NAMES[0], java_schema, - num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE, ) sql_schema = await lancedb.TableSchema.from_class( @@ -629,7 +625,6 @@ async def app_main() -> None: LANCE_DB, LANCE_TABLE_NAMES[1], sql_schema, - num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE, ) yaml_schema = await lancedb.TableSchema.from_class( @@ -640,7 +635,6 @@ async def app_main() -> None: LANCE_DB, LANCE_TABLE_NAMES[2], yaml_schema, - num_transactions_before_optimize=_NUM_TXN_BEFORE_OPTIMIZE, ) project_root = coco.use_context(PROJECT_ROOT) diff --git a/src/java_codebase_rag/lance_optimize.py b/src/java_codebase_rag/lance_optimize.py index 520de9a..f1d21ba 100644 --- a/src/java_codebase_rag/lance_optimize.py +++ b/src/java_codebase_rag/lance_optimize.py @@ -1,21 +1,20 @@ """Serialized post-flow LanceDB optimize with commit-conflict retry. -cocoindex 1.0.7 schedules ``table.optimize()`` (a LanceDB **Rewrite**/compaction -transaction) as a *background* ``asyncio`` task that races concurrent -``table.delete()`` (**Delete**) transactions emitted by later mutation batches. -LanceDB does not allow a Rewrite to commit concurrently with a Delete -(upstream lancedb#1504 — "We do not support concurrent deletes right now"), -which surfaces as a flood of:: +Historically (cocoindex 1.0.7) this existed because cocoindex scheduled +``table.optimize()`` (a LanceDB **Rewrite**/compaction) as a *background* +``asyncio`` task that raced concurrent ``table.delete()`` (**Delete**) +transactions — LanceDB does not allow a Rewrite to commit concurrently with a +Delete (upstream lancedb#1504), surfacing as a flood of:: RuntimeError: lance error: Retryable commit conflict for version N: \ This Rewrite transaction was preempted by concurrent transaction Delete ... -To eliminate the race, the flow (``java_index_flow_lancedb.py``) disables the -in-flight background optimize entirely by raising -``num_transactions_before_optimize`` to a value that is effectively never -reached. This module then performs a *single*, serialized optimize after the -flow returns (exit 0 → no concurrent writers), retrying the rare residual -commit conflict that two internal compaction passes can still produce. +cocoindex >=1.0.15 made optimize **inline and stats-driven** (only compacts +when small fragments accumulate, inside the merge_insert commit path), so the +race is gone and the flow no longer disables anything. This module still runs a +*single*, serialized optimize after the flow returns (exit 0 → no concurrent +writers) as a clean final compaction + scalar/FTS index build, retrying the rare +residual commit conflict that two internal compaction passes can still produce. """ from __future__ import annotations