feat(index): consolidate bitmap segments and unindexed data on optimize

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "8.0.0-beta.11"
+current_version = "8.1.0-beta.0"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",

.github/workflows/build_linux_wheel/action.yml

@@ -3,7 +3,7 @@ name: build-linux-wheel
 description: "Build a manylinux wheel for lance"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"

.github/workflows/build_mac_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"

.github/workflows/build_windows_wheel/action.yml

@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"

.github/workflows/java-publish.yml

@@ -28,10 +28,24 @@ permissions:
   contents: read
 
 jobs:
-  linux-arm64:
-    name: Build on Linux Arm64
-    runs-on: ubuntu-24.04-arm64-8x
+  build-linux:
+    name: Build on Linux ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86-64
+            runner: ubuntu-24.04
+            docker_platform: linux/amd64
+            protoc_arch: x86_64
+            artifact: liblance_jni_linux_x86_64.zip
+          - arch: arm64
+            runner: ubuntu-24.04-arm64-8x
+            docker_platform: linux/arm64
+            protoc_arch: aarch_64
+            artifact: liblance_jni_linux_arm_64.zip
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -41,9 +55,9 @@ jobs:
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
       - name: Check glibc version outside docker
         run: ldd --version
-      - name: Build and run in Debian 10 Arm64 container
+      - name: Build and run in Debian 10 container
         run: |
-          docker run --platform linux/arm64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
+          docker run --platform ${{ matrix.docker_platform }} -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
 
             set -ex
             # Update sources.list to use archive repositories for Debian 10 (EOL)
@@ -81,7 +95,7 @@ jobs:
               unzip
 
             # https://github.com/databendlabs/databend/issues/8035
-            PROTOC_ZIP=protoc-3.15.0-linux-aarch_64.zip
+            PROTOC_ZIP=protoc-3.15.0-linux-${{ matrix.protoc_arch }}.zip
             curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP
             unzip -o \$PROTOC_ZIP -d /usr/local
             rm -f \$PROTOC_ZIP
@@ -102,101 +116,44 @@ jobs:
           "
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: liblance_jni_linux_arm_64.zip
+          name: ${{ matrix.artifact }}
           path: java/lance-jni/target/release/liblance_jni.so
           retention-days: 1
           if-no-files-found: error
-  linux-x86:
-    name: Build on Linux x86-64
-    runs-on: ubuntu-24.04
+  build-macos:
+    name: Build on MacOS Arm64
+    runs-on: warp-macos-14-arm64-6x
     timeout-minutes: 60
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-      - name: Check glibc version outside docker
-        run: ldd --version
-      - name: Build and run in Debian 10 X86-64 container
-        run: |
-          docker run --platform linux/amd64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
-
-            set -ex
-            # Update sources.list to use archive repositories for Debian 10 (EOL)
-            echo 'deb http://archive.debian.org/debian/ buster main' > /etc/apt/sources.list
-            echo 'deb http://archive.debian.org/debian-security buster/updates main' >> /etc/apt/sources.list
-            echo 'deb http://archive.debian.org/debian/ buster-updates main' >> /etc/apt/sources.list
-            apt-get update
-
-            DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \
-              apt-transport-https \
-              ca-certificates \
-              curl \
-              gpg \
-              bash \
-              less \
-              openssl \
-              libssl-dev \
-              pkg-config \
-              libsqlite3-dev \
-              libsqlite3-0 \
-              libreadline-dev \
-              git \
-              cmake \
-              dh-autoreconf \
-              clang \
-              g++ \
-              libc++-dev \
-              libc++abi-dev \
-              libprotobuf-dev \
-              libncurses5-dev \
-              libncursesw5-dev \
-              libudev-dev \
-              libhidapi-dev \
-              zip \
-              unzip
-
-            # https://github.com/databendlabs/databend/issues/8035
-            PROTOC_ZIP=protoc-3.15.0-linux-x86_64.zip
-            curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP
-            unzip -o \$PROTOC_ZIP -d /usr/local
-            rm -f \$PROTOC_ZIP
-            protoc --version
-
-            curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
-            source \$HOME/.cargo/env
-            cargo --version
-
-            cd java/lance-jni
-
-            # https://github.com/rustls/rustls/issues/1967
-            export CC=clang
-            export CXX=clang++
-            ldd --version
-
-            cargo build --release
-          "
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07
+      - name: Install dependencies
+        run: brew install protobuf
+      - name: Build native lib
+        working-directory: java/lance-jni
+        run: cargo build --release
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: liblance_jni_linux_x86_64.zip
-          path: java/lance-jni/target/release/liblance_jni.so
+          name: liblance_jni_darwin_aarch64.zip
+          path: java/lance-jni/target/release/liblance_jni.dylib
           retention-days: 1
           if-no-files-found: error
-  macos-arm64:
-    name: Build on MacOS Arm64 and release
-    runs-on: warp-macos-14-arm64-6x
-    timeout-minutes: 60
+  publish:
+    name: Publish Java packages
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     needs:
-      - linux-arm64
-      - linux-x86
+      - build-linux
+      - build-macos
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Set up Java 11
         uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
@@ -208,18 +165,16 @@ jobs:
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07
-      - name: Install dependencies
-        run: |
-          brew install protobuf
-          brew install gpg
-      - name: Download artifact
+      - name: Download artifacts
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
       - name: Copy native libs
         run: |
-          mkdir -p ./java/target/classes/nativelib/linux-x86-64 ./java/target/classes/nativelib/linux-aarch64
+          mkdir -p ./java/target/classes/nativelib/linux-x86-64 \
+            ./java/target/classes/nativelib/linux-aarch64 \
+            ./java/target/classes/nativelib/darwin-aarch64
           cp ./liblance_jni_linux_x86_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-x86-64/liblance_jni.so
           cp ./liblance_jni_linux_arm_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-aarch64/liblance_jni.so
+          cp ./liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./java/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib
       - name: Set github
         run: |
           git config --global user.email "Lance Github Runner"
@@ -230,7 +185,7 @@ jobs:
           inputs.mode == 'dry_run'
         working-directory: java
         run: |
-          mvn --batch-mode -DskipTests -Drust.release.build=true package
+          mvn --batch-mode -DskipTests -Dskip.build.jni=true package
       - name: Publish with Java 11
         if: |
           github.event_name == 'release' ||
@@ -240,14 +195,14 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh -P shade-jar
+          mvn --batch-mode -DskipTests -Dskip.build.jni=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
-    needs: [linux-arm64, linux-x86, macos-arm64]
+    needs: [build-linux, build-macos, publish]
     if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
     permissions:
       contents: read

.github/workflows/pypi-publish.yml

@@ -35,7 +35,7 @@ jobs:
     name: Python Linux 3.${{ matrix.python-minor-version }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
         config:
           - platform: x86_64
             manylinux: "2_17"
@@ -101,7 +101,7 @@ jobs:
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
         config:
           - target: aarch64-apple-darwin
             runner: warp-macos-14-arm64-6x
@@ -152,7 +152,7 @@ jobs:
     runs-on: windows-latest-4x
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:

.github/workflows/python.yml

@@ -97,7 +97,7 @@ jobs:
     timeout-minutes: 45
     strategy:
       matrix:
-        python-minor-version: ["9", "13"]
+        python-minor-version: ["10", "13"]
     name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
     runs-on: "ubuntu-24.04-4x"
     defaults:

CONTRIBUTING.md

@@ -19,7 +19,7 @@ If you have any questions, please join our [Discord](https://discord.gg/zMM32dvN
 Currently Lance is implemented in Rust and comes with a Python wrapper. So you'll want to make sure you setup both.
 
 1. Install Rust: https://www.rust-lang.org/tools/install
-2. Install Python 3.9+: https://www.python.org/downloads/
+2. Install Python 3.10+: https://www.python.org/downloads/
 3. Install protoctol buffers: https://grpc.io/docs/protoc-installation/ (make sure you have version 3.20 or higher)
 4. Install commit hooks:
     a. Install pre-commit: https://pre-commit.com/#install