[Issue #1325] fix bugs of rockset index

.github/workflows/build-rockset.yml

@@ -0,0 +1,56 @@
+name: Build Rockset Module
+
+on:
+  pull_request:
+    paths:
+      - 'cpp/pixels-index/pixels-index-rockset/**'
+      - '.github/workflows/build-rockset.yml'
+  workflow_dispatch:
+
+jobs:
+  build-rockset:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+
+    env:
+      ROCKSET_MODULE_DIR: cpp/pixels-index/pixels-index-rockset
+      AWS_SDK_VERSION: 1.11.578
+      AWS_INSTALL_PREFIX: ${{ github.workspace }}/cpp/pixels-index/pixels-index-rockset/.cache/aws-sdk-install
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Java 8
+        uses: actions/setup-java@v4
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+
+      - name: Set PIXELS_HOME
+        run: |
+          export PIXELS_HOME="${{ github.workspace }}/pixels_home"
+          mkdir -p "$PIXELS_HOME/lib"
+          echo "PIXELS_HOME=$PIXELS_HOME" >> "$GITHUB_ENV"
+
+      - name: Cache AWS SDK
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.ROCKSET_MODULE_DIR }}/.deps/aws-sdk-cpp
+            ${{ env.ROCKSET_MODULE_DIR }}/.cache/aws-sdk-install
+          key: ${{ runner.os }}-rockset-aws-sdk-${{ env.AWS_SDK_VERSION }}-${{ hashFiles('cpp/pixels-index/pixels-index-rockset/ci-build.sh', '.github/workflows/build-rockset.yml') }}
+
+      - name: Cache rocksdb-cloud
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.ROCKSET_MODULE_DIR }}/build/deps
+          key: ${{ runner.os }}-rockset-cloud-${{ hashFiles('cpp/pixels-index/pixels-index-rockset/CMakeLists.txt', 'cpp/pixels-index/pixels-index-rockset/cmake/**', 'cpp/pixels-index/pixels-index-rockset/include/jni/**', 'cpp/pixels-index/pixels-index-rockset/lib/jni/**', 'cpp/pixels-index/pixels-index-rockset/ci-build.sh') }}
+
+      - name: Build pixels-index-rockset
+        working-directory: ${{ env.ROCKSET_MODULE_DIR }}
+        run: |
+          chmod +x ./ci-build.sh
+          ./ci-build.sh

cpp/pixels-index/pixels-index-rockset/CMakeLists.txt

@@ -4,6 +4,10 @@ project(pixels_rockset_jni)
 include(ExternalProject)
 include(ProcessorCount)
 
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
 # 0. Get core count
 ProcessorCount(CORES)
 if(CORES EQUAL 0)
@@ -26,34 +30,61 @@ endif()
 
 # 3. Configure rocksdb-cloud as external project
 set(ROCKSDB_CLOUD_GIT_REPOSITORY https://github.com/rockset/rocksdb-cloud.git)
-set(ROCKSDB_CLOUD_GIT_TAG master)
+set(ROCKSDB_CLOUD_GIT_TAG v6.7.3)
+set(AWS_SDK_PREFIX $ENV{AWS_INSTALL_PREFIX})
+if(NOT AWS_SDK_PREFIX)
+    set(AWS_SDK_PREFIX /usr/local)
+endif()
+set(AWS_SDK_INCLUDE_DIR ${AWS_SDK_PREFIX}/include)
+set(AWS_SDK_LIB_DIR ${AWS_SDK_PREFIX}/lib)
+set(AWS_SDK_LIB64_DIR ${AWS_SDK_PREFIX}/lib64)
+set(ROCKSDB_CLOUD_COMPAT_HEADER
+    ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RocksDBCloudCompat.h)
 set(ROCKSDB_CMAKE_ARGS
     -DCMAKE_BUILD_TYPE=Release
+    -DCMAKE_CXX_STANDARD=11
+    -DCMAKE_CXX_STANDARD_REQUIRED=ON
+    -DCMAKE_CXX_EXTENSIONS=OFF
     -DWITH_GFLAGS=OFF
     -DWITH_TESTS=OFF
     -DWITH_BENCHMARK_TOOLS=OFF
     -DWITH_CORE_TOOLS=OFF
+    -DWITH_TOOLS=OFF
     -DFAIL_ON_WARNINGS=OFF
     -DUSE_RTTI=1
-    -DCMAKE_CXX_FLAGS=-frtti
+    -DCMAKE_PREFIX_PATH=${AWS_SDK_PREFIX}
+    -DCMAKE_CXX_FLAGS=-frtti\ -DUSE_AWS\ -I${AWS_SDK_INCLUDE_DIR}\ -include\ ${ROCKSDB_CLOUD_COMPAT_HEADER}
+    -DCMAKE_SHARED_LINKER_FLAGS=-L${AWS_SDK_LIB_DIR}\ -L${AWS_SDK_LIB64_DIR}
+    -DCMAKE_EXE_LINKER_FLAGS=-L${AWS_SDK_LIB_DIR}\ -L${AWS_SDK_LIB64_DIR}
     -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/deps/rocksdb_cloud-install
     -DWITH_JNI=OFF
     -DWITH_SNAPPY=OFF
     -DWITH_ZLIB=ON
     -DWITH_BZ2=OFF
-    -DWITH_LZ4=OFF
+    -DWITH_LZ4=ON
     -DWITH_ZSTD=OFF
     -DWITH_LIBURING=ON
     -DWITH_AWS=ON
 )
 
+if(CMAKE_C_COMPILER)
+    list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
+endif()
+
+if(CMAKE_CXX_COMPILER)
+    list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
+endif()
+
 # 4. Download and build rocksdb-cloud
 ExternalProject_Add(rocksdb_cloud
     PREFIX ${CMAKE_CURRENT_BINARY_DIR}/deps
     GIT_REPOSITORY ${ROCKSDB_CLOUD_GIT_REPOSITORY}
     GIT_TAG ${ROCKSDB_CLOUD_GIT_TAG}
     GIT_SHALLOW true
     UPDATE_COMMAND ""
+    PATCH_COMMAND ${CMAKE_COMMAND}
+        -DROCKSDB_CLOUD_SOURCE_DIR=<SOURCE_DIR>
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/PatchRocksDBCloud.cmake
     CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS}
     BUILD_BYPRODUCTS
         ${CMAKE_CURRENT_BINARY_DIR}/deps/rocksdb_cloud-install/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}
@@ -65,6 +96,7 @@ ExternalProject_Add(rocksdb_cloud
 ExternalProject_Get_Property(rocksdb_cloud INSTALL_DIR)
 set(ROCKSDB_CLOUD_INCLUDE_DIR ${INSTALL_DIR}/include)
 set(ROCKSDB_CLOUD_LIBRARY ${INSTALL_DIR}/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(ROCKSDB_CLOUD_LIBRARY_GLOB ${INSTALL_DIR}/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}*)
 file(MAKE_DIRECTORY ${ROCKSDB_CLOUD_INCLUDE_DIR})
 
 # 6. Create imported target for rocksdb
@@ -97,15 +129,36 @@ add_library(pixels-index-rockset SHARED
 
 add_dependencies(pixels-index-rockset rocksdb_cloud)
 
+set_target_properties(pixels-index-rockset PROPERTIES
+    BUILD_RPATH "${INSTALL_DIR}/lib"
+    INSTALL_RPATH "\$ORIGIN"
+)
+
+target_compile_definitions(pixels-index-rockset
+    PRIVATE ROCKSDB_NAMESPACE=rocksdb
+)
+
 # 10. ink RocksDB and its dependencies
 target_link_libraries(pixels-index-rockset
     RocksDB::rocksdb
     z
+    lz4
     curl
     pthread
     dl
 )
 
 # 11. Install pixels-index-rockset to ${PIXELS_HOME}/lib
 install(TARGETS pixels-index-rockset
-        LIBRARY DESTINATION ${PIXELS_HOME}/lib)
+        LIBRARY DESTINATION ${PIXELS_HOME}/lib)
+install(FILES ${ROCKSDB_CLOUD_LIBRARY}
+        DESTINATION ${PIXELS_HOME}/lib)
+install(CODE "
+  file(GLOB ROCKSDB_RUNTIME_LIBS \"${ROCKSDB_CLOUD_LIBRARY_GLOB}\")
+  foreach(RUNTIME_LIB IN LISTS ROCKSDB_RUNTIME_LIBS)
+    get_filename_component(RUNTIME_LIB_NAME \"\${RUNTIME_LIB}\" NAME)
+    if(NOT RUNTIME_LIB_NAME STREQUAL \"librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}\")
+      file(INSTALL DESTINATION \"${PIXELS_HOME}/lib\" TYPE FILE FILES \"\${RUNTIME_LIB}\")
+    endif()
+  endforeach()
+")

cpp/pixels-index/pixels-index-rockset/README.md

@@ -5,12 +5,58 @@ This is the corresponding C++ module for `RocksetIndex`.
 It implements the JNI methods for pixels-index-rockset to operate Rockset (RocksDB Cloud).
 
 ### Build Instructions
+This module is built and tested with:
+
+- GCC/G++ with C++11 support (version 4.8 or later)
+- RocksDB Cloud `v6.7.3`
+- AWS SDK for C++ installed under `/usr/local`
+
+The RocksDB Cloud version is pinned in `CMakeLists.txt` as `ROCKSDB_CLOUD_GIT_TAG v6.7.3`.
+Do not switch it to `master`: newer RocksDB headers are not ABI/API compatible with this module.
+
 1. We download and install rocksdb-cloud automatically in the CMakeLists of pixels-index-rockset.
    However, the compilation process of rocksdb-cloud assumes that the AWS c++ SDK is installed in
    the default location of /usr/local. You can follow the steps listed here https://github.com/aws/aws-sdk-cpp (version 1.7.325 or later).
-   Here are the instructions for your reference:
+   Following the RocksDB Linux - Ubuntu instructions, install the prerequisite packages first:
+```bash
+sudo apt-get install \
+  libgflags-dev \
+  libsnappy-dev \
+  zlib1g-dev \
+  libbz2-dev \
+  liblz4-dev \
+  libzstd-dev \
+  libcurl4-openssl-dev
+```
+   Or on CentOS / RHEL, prepare the prerequisites like this:
+```bash
+# GCC/G++ with C++11 support
+sudo yum install gcc48-c++
+
+# gflags
+git clone https://github.com/gflags/gflags.git
+cd gflags
+git checkout v2.0
+./configure && make && sudo make install
+# If installed to /usr/local, export CPATH=/usr/local/include:$CPATH
+# and LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH
+
+# snappy / zlib / bzip2 / lz4 / optional ASAN
+sudo yum install snappy snappy-devel
+sudo yum install zlib zlib-devel
+sudo yum install bzip2 bzip2-devel
+sudo yum install lz4-devel
+sudo yum install libasan
+
+# zstd
+wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+tar zxvf zstd-1.1.3.tar.gz
+cd zstd-1.1.3
+make && sudo make install
+```
+   Here are the AWS SDK build and install commands for reference:
 ```bash
-sudo apt install libcurl4-openssl-dev # libcurl is required by aws-sdk-cpp
 git clone https://github.com/aws/aws-sdk-cpp
 cd aws-sdk-cpp
 git checkout 1.11.578 ## we use this version by default
@@ -27,24 +73,49 @@ cmake --build . --config=Release
 sudo cmake --install . --config=Release
 ```
 
-2. Enter the cpp source directory of pixels-index-rockset, 
-ensure the `PIXELS_HOME` and `JAVA_HOME` environment variables are set correctly, then execute:
+2. Use the CI/bootstrap build script. It supports Ubuntu and CentOS/RHEL-like systems, installs the documented system dependencies, installs AWS SDK for C++ under `/usr/local` when missing, and then runs the native build.
+Provide Java separately in the environment before running it:
+```bash
+cd cpp/pixels-index/pixels-index-rockset
+./ci-build.sh
+```
+The script checks and/or prepares:
+- `cmake`, `make`, `git`
+- `PIXELS_HOME`
+- `JAVA_HOME` and JNI headers
+- `gcc` and `g++` in `PATH`, with version `4.8` or later
+- RocksDB build dependencies for Linux, including Ubuntu and CentOS/RHEL variants: `gflags`, `snappy`, `zlib`, `bzip2`, `lz4`, `zstd`, `libcurl`
+- AWS SDK for C++ installed under `/usr/local`
+After a successful build, it also installs `libpixels-index-rockset.so` and the required `librocksdb.so*` runtime libraries into `$PIXELS_HOME/lib`.
+
+If you need to run the native build commands manually after the environment is ready, the script executes the equivalent workflow:
 ```bash
+export CC=/path/to/gcc
+export CXX=/path/to/g++
 mkdir build && cd build
 cmake .. # in the output of this command, check whether the correct JNI version is used
 make
 ```
 
-3. After completing the above steps, if everything works correctly, you can find the generated `libpixels-index-rockset.so` in the `$PIXELS_HOME/lib` directory.
-This shared library is to be loaded and bind to the Java code in `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java`.
+3. After completing the above steps, you can find the generated `libpixels-index-rockset.so` in the local `build/` directory.
+
+4. If everything works correctly, you can find the installed `libpixels-index-rockset.so` and `librocksdb.so*` in the `$PIXELS_HOME/lib` directory.
+This shared library is loaded by the Java code under `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset`.
+
+You can verify that the expected RocksDB Cloud and LZ4 dependencies are linked with:
+```bash
+ldd "$PIXELS_HOME/lib/libpixels-index-rockset.so"
+ldd build/deps/rocksdb_cloud-install/lib/librocksdb.so | grep lz4
+```
 
-If you modified the native methods in `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java`, regenerate the `RocksetJni.h` file using:
+If you modified the native methods in the Java JNI classes under
+`PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni`,
+regenerate the corresponding JNI header files using:
 ```bash
 # enter the cpp source path of pixels-index-rockset
 cd cpp/pixels-index/pixels-index-rockset/
-#regenerate the JNI header file
-javac -cp . -h ./include/ io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java
-# rename the header file
-mv ./include/io_pixelsdb_pixels_index_rocksdb_RocksetIndex.h ./include/RocksetJni.h
+# regenerate JNI header files from compiled Java classes
+javac -h ./include/jni \
+  ../../../pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/*.java
 ```
-And redo the above steps to build the `libpixels-index-rockset.so` library.
+And redo the above steps to build the `libpixels-index-rockset.so` library.