diff --git a/.github/workflows/build-rockset.yml b/.github/workflows/build-rockset.yml new file mode 100644 index 0000000000..7d4afeed3b --- /dev/null +++ b/.github/workflows/build-rockset.yml @@ -0,0 +1,56 @@ +name: Build Rockset Module + +on: + pull_request: + paths: + - 'cpp/pixels-index/pixels-index-rockset/**' + - '.github/workflows/build-rockset.yml' + workflow_dispatch: + +jobs: + build-rockset: + runs-on: ubuntu-22.04 + permissions: + contents: read + + env: + ROCKSET_MODULE_DIR: cpp/pixels-index/pixels-index-rockset + AWS_SDK_VERSION: 1.11.578 + AWS_INSTALL_PREFIX: ${{ github.workspace }}/cpp/pixels-index/pixels-index-rockset/.cache/aws-sdk-install + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java 8 + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'temurin' + + - name: Set PIXELS_HOME + run: | + export PIXELS_HOME="${{ github.workspace }}/pixels_home" + mkdir -p "$PIXELS_HOME/lib" + echo "PIXELS_HOME=$PIXELS_HOME" >> "$GITHUB_ENV" + + - name: Cache AWS SDK + uses: actions/cache@v4 + with: + path: | + ${{ env.ROCKSET_MODULE_DIR }}/.deps/aws-sdk-cpp + ${{ env.ROCKSET_MODULE_DIR }}/.cache/aws-sdk-install + key: ${{ runner.os }}-rockset-aws-sdk-${{ env.AWS_SDK_VERSION }}-${{ hashFiles('cpp/pixels-index/pixels-index-rockset/ci-build.sh', '.github/workflows/build-rockset.yml') }} + + - name: Cache rocksdb-cloud + uses: actions/cache@v4 + with: + path: | + ${{ env.ROCKSET_MODULE_DIR }}/build/deps + key: ${{ runner.os }}-rockset-cloud-${{ hashFiles('cpp/pixels-index/pixels-index-rockset/CMakeLists.txt', 'cpp/pixels-index/pixels-index-rockset/cmake/**', 'cpp/pixels-index/pixels-index-rockset/include/jni/**', 'cpp/pixels-index/pixels-index-rockset/lib/jni/**', 'cpp/pixels-index/pixels-index-rockset/ci-build.sh') }} + + - name: Build pixels-index-rockset + working-directory: ${{ env.ROCKSET_MODULE_DIR }} + run: | + chmod +x ./ci-build.sh + ./ci-build.sh diff --git a/cpp/pixels-index/pixels-index-rockset/CMakeLists.txt b/cpp/pixels-index/pixels-index-rockset/CMakeLists.txt index 9b50ca9767..7a3b47ada5 100644 --- a/cpp/pixels-index/pixels-index-rockset/CMakeLists.txt +++ b/cpp/pixels-index/pixels-index-rockset/CMakeLists.txt @@ -4,6 +4,10 @@ project(pixels_rockset_jni) include(ExternalProject) include(ProcessorCount) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + # 0. Get core count ProcessorCount(CORES) if(CORES EQUAL 0) @@ -26,27 +30,51 @@ endif() # 3. Configure rocksdb-cloud as external project set(ROCKSDB_CLOUD_GIT_REPOSITORY https://github.com/rockset/rocksdb-cloud.git) -set(ROCKSDB_CLOUD_GIT_TAG master) +set(ROCKSDB_CLOUD_GIT_TAG v6.7.3) +set(AWS_SDK_PREFIX $ENV{AWS_INSTALL_PREFIX}) +if(NOT AWS_SDK_PREFIX) + set(AWS_SDK_PREFIX /usr/local) +endif() +set(AWS_SDK_INCLUDE_DIR ${AWS_SDK_PREFIX}/include) +set(AWS_SDK_LIB_DIR ${AWS_SDK_PREFIX}/lib) +set(AWS_SDK_LIB64_DIR ${AWS_SDK_PREFIX}/lib64) +set(ROCKSDB_CLOUD_COMPAT_HEADER + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/RocksDBCloudCompat.h) set(ROCKSDB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_STANDARD=11 + -DCMAKE_CXX_STANDARD_REQUIRED=ON + -DCMAKE_CXX_EXTENSIONS=OFF -DWITH_GFLAGS=OFF -DWITH_TESTS=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_CORE_TOOLS=OFF + -DWITH_TOOLS=OFF -DFAIL_ON_WARNINGS=OFF -DUSE_RTTI=1 - -DCMAKE_CXX_FLAGS=-frtti + -DCMAKE_PREFIX_PATH=${AWS_SDK_PREFIX} + -DCMAKE_CXX_FLAGS=-frtti\ -DUSE_AWS\ -I${AWS_SDK_INCLUDE_DIR}\ -include\ ${ROCKSDB_CLOUD_COMPAT_HEADER} + -DCMAKE_SHARED_LINKER_FLAGS=-L${AWS_SDK_LIB_DIR}\ -L${AWS_SDK_LIB64_DIR} + -DCMAKE_EXE_LINKER_FLAGS=-L${AWS_SDK_LIB_DIR}\ -L${AWS_SDK_LIB64_DIR} -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/deps/rocksdb_cloud-install -DWITH_JNI=OFF -DWITH_SNAPPY=OFF -DWITH_ZLIB=ON -DWITH_BZ2=OFF - -DWITH_LZ4=OFF + -DWITH_LZ4=ON -DWITH_ZSTD=OFF -DWITH_LIBURING=ON -DWITH_AWS=ON ) +if(CMAKE_C_COMPILER) + list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}) +endif() + +if(CMAKE_CXX_COMPILER) + list(APPEND ROCKSDB_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) +endif() + # 4. Download and build rocksdb-cloud ExternalProject_Add(rocksdb_cloud PREFIX ${CMAKE_CURRENT_BINARY_DIR}/deps @@ -54,6 +82,9 @@ ExternalProject_Add(rocksdb_cloud GIT_TAG ${ROCKSDB_CLOUD_GIT_TAG} GIT_SHALLOW true UPDATE_COMMAND "" + PATCH_COMMAND ${CMAKE_COMMAND} + -DROCKSDB_CLOUD_SOURCE_DIR= + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/PatchRocksDBCloud.cmake CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/deps/rocksdb_cloud-install/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX} @@ -65,6 +96,7 @@ ExternalProject_Add(rocksdb_cloud ExternalProject_Get_Property(rocksdb_cloud INSTALL_DIR) set(ROCKSDB_CLOUD_INCLUDE_DIR ${INSTALL_DIR}/include) set(ROCKSDB_CLOUD_LIBRARY ${INSTALL_DIR}/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(ROCKSDB_CLOUD_LIBRARY_GLOB ${INSTALL_DIR}/lib/librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}*) file(MAKE_DIRECTORY ${ROCKSDB_CLOUD_INCLUDE_DIR}) # 6. Create imported target for rocksdb @@ -97,10 +129,20 @@ add_library(pixels-index-rockset SHARED add_dependencies(pixels-index-rockset rocksdb_cloud) +set_target_properties(pixels-index-rockset PROPERTIES + BUILD_RPATH "${INSTALL_DIR}/lib" + INSTALL_RPATH "\$ORIGIN" +) + +target_compile_definitions(pixels-index-rockset + PRIVATE ROCKSDB_NAMESPACE=rocksdb +) + # 10. ink RocksDB and its dependencies target_link_libraries(pixels-index-rockset RocksDB::rocksdb z + lz4 curl pthread dl @@ -108,4 +150,15 @@ target_link_libraries(pixels-index-rockset # 11. Install pixels-index-rockset to ${PIXELS_HOME}/lib install(TARGETS pixels-index-rockset - LIBRARY DESTINATION ${PIXELS_HOME}/lib) \ No newline at end of file + LIBRARY DESTINATION ${PIXELS_HOME}/lib) +install(FILES ${ROCKSDB_CLOUD_LIBRARY} + DESTINATION ${PIXELS_HOME}/lib) +install(CODE " + file(GLOB ROCKSDB_RUNTIME_LIBS \"${ROCKSDB_CLOUD_LIBRARY_GLOB}\") + foreach(RUNTIME_LIB IN LISTS ROCKSDB_RUNTIME_LIBS) + get_filename_component(RUNTIME_LIB_NAME \"\${RUNTIME_LIB}\" NAME) + if(NOT RUNTIME_LIB_NAME STREQUAL \"librocksdb${CMAKE_SHARED_LIBRARY_SUFFIX}\") + file(INSTALL DESTINATION \"${PIXELS_HOME}/lib\" TYPE FILE FILES \"\${RUNTIME_LIB}\") + endif() + endforeach() +") diff --git a/cpp/pixels-index/pixels-index-rockset/README.md b/cpp/pixels-index/pixels-index-rockset/README.md index 063e1fde75..74f7096d3b 100644 --- a/cpp/pixels-index/pixels-index-rockset/README.md +++ b/cpp/pixels-index/pixels-index-rockset/README.md @@ -5,12 +5,58 @@ This is the corresponding C++ module for `RocksetIndex`. It implements the JNI methods for pixels-index-rockset to operate Rockset (RocksDB Cloud). ### Build Instructions +This module is built and tested with: + +- GCC/G++ with C++11 support (version 4.8 or later) +- RocksDB Cloud `v6.7.3` +- AWS SDK for C++ installed under `/usr/local` + +The RocksDB Cloud version is pinned in `CMakeLists.txt` as `ROCKSDB_CLOUD_GIT_TAG v6.7.3`. +Do not switch it to `master`: newer RocksDB headers are not ABI/API compatible with this module. + 1. We download and install rocksdb-cloud automatically in the CMakeLists of pixels-index-rockset. However, the compilation process of rocksdb-cloud assumes that the AWS c++ SDK is installed in the default location of /usr/local. You can follow the steps listed here https://github.com/aws/aws-sdk-cpp (version 1.7.325 or later). - Here are the instructions for your reference: + Following the RocksDB Linux - Ubuntu instructions, install the prerequisite packages first: +```bash +sudo apt-get install \ + libgflags-dev \ + libsnappy-dev \ + zlib1g-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + libcurl4-openssl-dev +``` + Or on CentOS / RHEL, prepare the prerequisites like this: +```bash +# GCC/G++ with C++11 support +sudo yum install gcc48-c++ + +# gflags +git clone https://github.com/gflags/gflags.git +cd gflags +git checkout v2.0 +./configure && make && sudo make install +# If installed to /usr/local, export CPATH=/usr/local/include:$CPATH +# and LIBRARY_PATH=/usr/local/lib:$LIBRARY_PATH + +# snappy / zlib / bzip2 / lz4 / optional ASAN +sudo yum install snappy snappy-devel +sudo yum install zlib zlib-devel +sudo yum install bzip2 bzip2-devel +sudo yum install lz4-devel +sudo yum install libasan + +# zstd +wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz +mv v1.1.3.tar.gz zstd-1.1.3.tar.gz +tar zxvf zstd-1.1.3.tar.gz +cd zstd-1.1.3 +make && sudo make install +``` + Here are the AWS SDK build and install commands for reference: ```bash -sudo apt install libcurl4-openssl-dev # libcurl is required by aws-sdk-cpp git clone https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp git checkout 1.11.578 ## we use this version by default @@ -27,24 +73,49 @@ cmake --build . --config=Release sudo cmake --install . --config=Release ``` -2. Enter the cpp source directory of pixels-index-rockset, -ensure the `PIXELS_HOME` and `JAVA_HOME` environment variables are set correctly, then execute: +2. Use the CI/bootstrap build script. It supports Ubuntu and CentOS/RHEL-like systems, installs the documented system dependencies, installs AWS SDK for C++ under `/usr/local` when missing, and then runs the native build. +Provide Java separately in the environment before running it: +```bash +cd cpp/pixels-index/pixels-index-rockset +./ci-build.sh +``` +The script checks and/or prepares: +- `cmake`, `make`, `git` +- `PIXELS_HOME` +- `JAVA_HOME` and JNI headers +- `gcc` and `g++` in `PATH`, with version `4.8` or later +- RocksDB build dependencies for Linux, including Ubuntu and CentOS/RHEL variants: `gflags`, `snappy`, `zlib`, `bzip2`, `lz4`, `zstd`, `libcurl` +- AWS SDK for C++ installed under `/usr/local` +After a successful build, it also installs `libpixels-index-rockset.so` and the required `librocksdb.so*` runtime libraries into `$PIXELS_HOME/lib`. + +If you need to run the native build commands manually after the environment is ready, the script executes the equivalent workflow: ```bash +export CC=/path/to/gcc +export CXX=/path/to/g++ mkdir build && cd build cmake .. # in the output of this command, check whether the correct JNI version is used make ``` -3. After completing the above steps, if everything works correctly, you can find the generated `libpixels-index-rockset.so` in the `$PIXELS_HOME/lib` directory. -This shared library is to be loaded and bind to the Java code in `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java`. +3. After completing the above steps, you can find the generated `libpixels-index-rockset.so` in the local `build/` directory. + +4. If everything works correctly, you can find the installed `libpixels-index-rockset.so` and `librocksdb.so*` in the `$PIXELS_HOME/lib` directory. +This shared library is loaded by the Java code under `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset`. + +You can verify that the expected RocksDB Cloud and LZ4 dependencies are linked with: +```bash +ldd "$PIXELS_HOME/lib/libpixels-index-rockset.so" +ldd build/deps/rocksdb_cloud-install/lib/librocksdb.so | grep lz4 +``` -If you modified the native methods in `PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java`, regenerate the `RocksetJni.h` file using: +If you modified the native methods in the Java JNI classes under +`PIXELS_SRC/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni`, +regenerate the corresponding JNI header files using: ```bash # enter the cpp source path of pixels-index-rockset cd cpp/pixels-index/pixels-index-rockset/ -#regenerate the JNI header file -javac -cp . -h ./include/ io/pixelsdb/pixels/index/rocksdb/RocksetIndex.java -# rename the header file -mv ./include/io_pixelsdb_pixels_index_rocksdb_RocksetIndex.h ./include/RocksetJni.h +# regenerate JNI header files from compiled Java classes +javac -h ./include/jni \ + ../../../pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/*.java ``` -And redo the above steps to build the `libpixels-index-rockset.so` library. \ No newline at end of file +And redo the above steps to build the `libpixels-index-rockset.so` library. diff --git a/cpp/pixels-index/pixels-index-rockset/ci-build.sh b/cpp/pixels-index/pixels-index-rockset/ci-build.sh new file mode 100755 index 0000000000..6b1e28c45f --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/ci-build.sh @@ -0,0 +1,414 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +BUILD_DIR="${SCRIPT_DIR}/build" +AWS_SDK_VERSION="${AWS_SDK_VERSION:-1.11.578}" +AWS_INSTALL_PREFIX="${AWS_INSTALL_PREFIX:-/usr/local}" +WORK_DIR="${SCRIPT_DIR}/.deps" +CACHE_DIR="${SCRIPT_DIR}/.cache" +AWS_WORK_DIR="${WORK_DIR}/aws-sdk-cpp" +GFLAGS_WORK_DIR="${WORK_DIR}/gflags" +ZSTD_WORK_DIR="${WORK_DIR}/zstd" + +SUDO="" +if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then + SUDO="sudo" +fi + +note() { + echo "[pixels-index-rockset-ci] $*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +run() { + note "$*" + "$@" +} + +require_command() { + local cmd="$1" + command -v "$cmd" >/dev/null 2>&1 || die "Required command '$cmd' was not found in PATH." +} + +require_file() { + local path="$1" + local hint="${2:-}" + if [[ ! -f "$path" ]]; then + if [[ -n "$hint" ]]; then + die "Missing required file '$path'. $hint" + fi + die "Missing required file '$path'." + fi +} + +require_dir() { + local path="$1" + local hint="${2:-}" + if [[ ! -d "$path" ]]; then + if [[ -n "$hint" ]]; then + die "Missing required directory '$path'. $hint" + fi + die "Missing required directory '$path'." + fi +} + +version_ge() { + local current="$1" + local minimum="$2" + [[ "$(printf '%s\n%s\n' "$minimum" "$current" | sort -V | head -n1)" == "$minimum" ]] +} + +check_compiler() { + local compiler_path="$1" + local compiler_name="$2" + local compiler_version + + [[ -n "$compiler_path" ]] || die "Required compiler '$compiler_name' was not found in PATH." + [[ -x "$compiler_path" ]] || die "Compiler '$compiler_path' is not executable." + + compiler_version=$("$compiler_path" -dumpversion 2>/dev/null | cut -d. -f1,2) + [[ -n "$compiler_version" ]] || die "Unable to determine version for compiler '$compiler_path'." + + version_ge "$compiler_version" "4.8" || die \ + "Compiler '$compiler_path' is version $compiler_version. GCC/G++ 4.8 or later is required for C++11 support." +} + +find_aws_lib_dir() { + local candidate + for candidate in "${AWS_INSTALL_PREFIX}/lib" "${AWS_INSTALL_PREFIX}/lib64"; do + if [[ -f "${candidate}/libaws-cpp-sdk-core.so" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +find_library_dir() { + local lib_name="$1" + local candidate + for candidate in \ + /usr/lib \ + /usr/lib64 \ + /usr/local/lib \ + /usr/local/lib64 \ + /lib/x86_64-linux-gnu \ + /usr/lib/x86_64-linux-gnu \ + /lib/aarch64-linux-gnu \ + /usr/lib/aarch64-linux-gnu; do + if [[ -f "${candidate}/${lib_name}" ]]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +find_header_file() { + local header_path="$1" + local candidate + for candidate in \ + /usr/include \ + /usr/local/include \ + /usr/include/x86_64-linux-gnu \ + /usr/local/include/x86_64-linux-gnu \ + /usr/include/aarch64-linux-gnu \ + /usr/local/include/aarch64-linux-gnu; do + if [[ -f "${candidate}/${header_path}" ]]; then + echo "${candidate}/${header_path}" + return 0 + fi + done + return 1 +} + +check_system_dependency() { + local header="$1" + local library="$2" + local dependency_name="$3" + local install_hint="$4" + + find_header_file "$header" >/dev/null || die \ + "Missing header '$header' for ${dependency_name}. ${install_hint}" + find_library_dir "$library" >/dev/null || die \ + "Missing library '$library' for ${dependency_name}. ${install_hint}" +} + +check_linux_build_deps() { + check_system_dependency \ + "gflags/gflags.h" \ + "libgflags.so" \ + "gflags" \ + "On Ubuntu, install 'libgflags-dev'. On CentOS/RHEL, build gflags v2.0 from source and expose its include/lib paths through CPATH and LIBRARY_PATH if installed under /usr/local." + check_system_dependency \ + "snappy.h" \ + "libsnappy.so" \ + "snappy" \ + "On Ubuntu, install 'libsnappy-dev'. On CentOS/RHEL, install 'snappy snappy-devel'." + check_system_dependency \ + "zlib.h" \ + "libz.so" \ + "zlib" \ + "On Ubuntu, install 'zlib1g-dev'. On CentOS/RHEL, install 'zlib zlib-devel'." + check_system_dependency \ + "bzlib.h" \ + "libbz2.so" \ + "bzip2" \ + "On Ubuntu, install 'libbz2-dev'. On CentOS/RHEL, install 'bzip2 bzip2-devel'." + check_system_dependency \ + "lz4.h" \ + "liblz4.so" \ + "lz4" \ + "On Ubuntu, install 'liblz4-dev'. On CentOS/RHEL, install 'lz4-devel'." + check_system_dependency \ + "zstd.h" \ + "libzstd.so" \ + "zstandard" \ + "On Ubuntu, install 'libzstd-dev'. On CentOS/RHEL, build zstd from source or install the distro-provided development package if available." + check_system_dependency \ + "curl/curl.h" \ + "libcurl.so" \ + "libcurl" \ + "On Ubuntu, install 'libcurl4-openssl-dev'. On CentOS/RHEL, install the libcurl development package for your distribution." +} + +check_aws_sdk() { + local aws_include="${AWS_INSTALL_PREFIX}/include/aws/core/Aws.h" + local aws_lib_dir + + require_file \ + "$aws_include" \ + "Install AWS SDK for C++ to ${AWS_INSTALL_PREFIX}. See README.md for the exact commands." + + aws_lib_dir=$(find_aws_lib_dir) || die \ + "AWS SDK libraries were not found under ${AWS_INSTALL_PREFIX}/lib or ${AWS_INSTALL_PREFIX}/lib64. See README.md for the expected installation steps." + + require_file "${aws_lib_dir}/libaws-cpp-sdk-core.so" + require_file "${aws_lib_dir}/libaws-cpp-sdk-s3.so" + require_file "${aws_lib_dir}/libaws-cpp-sdk-transfer.so" + require_file "${aws_lib_dir}/libaws-cpp-sdk-kinesis.so" +} + +source_os_release() { + [[ -f /etc/os-release ]] || die "Cannot detect Linux distribution: /etc/os-release is missing." + # shellcheck disable=SC1091 + source /etc/os-release +} + +is_ubuntu_like() { + [[ "${ID:-}" == "ubuntu" ]] || [[ "${ID_LIKE:-}" == *"debian"* ]] +} + +is_rhel_like() { + [[ "${ID:-}" == "centos" ]] || [[ "${ID:-}" == "rhel" ]] || [[ "${ID:-}" == "rocky" ]] || [[ "${ID:-}" == "almalinux" ]] || [[ "${ID_LIKE:-}" == *"rhel"* ]] || [[ "${ID_LIKE:-}" == *"fedora"* ]] +} + +pick_pkg_manager() { + if command -v dnf >/dev/null 2>&1; then + echo "dnf" + return 0 + fi + if command -v yum >/dev/null 2>&1; then + echo "yum" + return 0 + fi + die "Neither dnf nor yum is available." +} + +install_ubuntu_packages() { + run ${SUDO} apt-get update + run ${SUDO} apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + pkg-config \ + libgflags-dev \ + libsnappy-dev \ + zlib1g-dev \ + libbz2-dev \ + liblz4-dev \ + libzstd-dev \ + libcurl4-openssl-dev +} + +install_rhel_packages() { + local pkg_manager + pkg_manager=$(pick_pkg_manager) + + run ${SUDO} "${pkg_manager}" install -y \ + gcc \ + gcc-c++ \ + make \ + cmake \ + git \ + curl \ + pkgconfig \ + snappy \ + snappy-devel \ + zlib \ + zlib-devel \ + bzip2 \ + bzip2-devel \ + lz4-devel \ + which +} + +ensure_gflags_rhel() { + if [[ -f /usr/include/gflags/gflags.h || -f /usr/local/include/gflags/gflags.h ]]; then + return 0 + fi + + mkdir -p "$WORK_DIR" + if [[ ! -d "$GFLAGS_WORK_DIR/.git" ]]; then + run git clone https://github.com/gflags/gflags.git "$GFLAGS_WORK_DIR" + fi + + ( + cd "$GFLAGS_WORK_DIR" + run git fetch --tags --force + run git checkout v2.0 + run ./configure + run make -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" + run ${SUDO} make install + ) + + export CPATH="/usr/local/include:${CPATH:-}" + export LIBRARY_PATH="/usr/local/lib:${LIBRARY_PATH:-}" + export LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH:-}" +} + +ensure_zstd_rhel() { + if [[ -f /usr/include/zstd.h || -f /usr/local/include/zstd.h ]]; then + return 0 + fi + + mkdir -p "$WORK_DIR" + if [[ ! -d "$ZSTD_WORK_DIR" ]]; then + ( + cd "$WORK_DIR" + run curl -L -o zstd-1.1.3.tar.gz https://github.com/facebook/zstd/archive/v1.1.3.tar.gz + run tar zxvf zstd-1.1.3.tar.gz + run mv zstd-1.1.3 "$ZSTD_WORK_DIR" + ) + fi + + ( + cd "$ZSTD_WORK_DIR" + run make -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" + run ${SUDO} make install + ) +} + +ensure_aws_sdk() { + mkdir -p "$WORK_DIR" "$CACHE_DIR" "$AWS_INSTALL_PREFIX" + + if [[ -f "${AWS_INSTALL_PREFIX}/include/aws/core/Aws.h" ]] && \ + [[ -f "${AWS_INSTALL_PREFIX}/lib/libaws-cpp-sdk-core.so" || -f "${AWS_INSTALL_PREFIX}/lib64/libaws-cpp-sdk-core.so" ]]; then + return 0 + fi + + if [[ ! -d "$AWS_WORK_DIR/.git" ]]; then + run git clone https://github.com/aws/aws-sdk-cpp "$AWS_WORK_DIR" + fi + + ( + cd "$AWS_WORK_DIR" + run git fetch --tags --force + run git checkout "${AWS_SDK_VERSION}" + run git submodule update --init --recursive + rm -rf build + mkdir -p build + cd build + run cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_ONLY=s3\;core\;transfer\;kinesis \ + -DENABLE_UNITY_BUILD=ON \ + -DENABLE_TESTING=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX="${AWS_INSTALL_PREFIX}" + run cmake --build . --config Release -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" + run ${SUDO} cmake --install . --config Release + ) +} + +ensure_linux_dependencies() { + source_os_release + + if is_ubuntu_like; then + install_ubuntu_packages + return 0 + fi + + if is_rhel_like; then + install_rhel_packages + ensure_gflags_rhel + ensure_zstd_rhel + return 0 + fi + + die "Unsupported Linux distribution '${ID:-unknown}'. This CI script supports Ubuntu and CentOS/RHEL-like systems." +} + +main() { + local cc_bin + local cxx_bin + + [[ -n "${PIXELS_HOME:-}" ]] || die "PIXELS_HOME is not set." + [[ -n "${JAVA_HOME:-}" ]] || note "JAVA_HOME is not set. Provide a JDK in the environment before running this script." + + ensure_linux_dependencies + ensure_aws_sdk + + if [[ -z "${JAVA_HOME:-}" ]] && command -v javac >/dev/null 2>&1; then + export JAVA_HOME + JAVA_HOME=$(dirname "$(dirname "$(readlink -f "$(command -v javac)")")") + note "Derived JAVA_HOME=${JAVA_HOME}" + fi + + require_command cmake + require_command make + require_command git + + cc_bin="${CC:-$(command -v gcc || true)}" + cxx_bin="${CXX:-$(command -v g++ || true)}" + + require_dir "$PIXELS_HOME" "Run './install.sh' first or point PIXELS_HOME to a valid Pixels installation directory." + require_dir "${JAVA_HOME}/include" "JAVA_HOME should point to a JDK, not a JRE." + require_file "${JAVA_HOME}/include/jni.h" "JAVA_HOME should point to a JDK with JNI headers." + + check_compiler "$cc_bin" "gcc" + check_compiler "$cxx_bin" "g++" + check_linux_build_deps + check_aws_sdk + + mkdir -p "$BUILD_DIR" + + note "Using PIXELS_HOME=$PIXELS_HOME" + note "Using JAVA_HOME=$JAVA_HOME" + note "Using CC=$cc_bin" + note "Using CXX=$cxx_bin" + note "Using AWS_INSTALL_PREFIX=$AWS_INSTALL_PREFIX" + note "Configuring build directory: $BUILD_DIR" + + ( + cd "$BUILD_DIR" + export CMAKE_PREFIX_PATH="${AWS_INSTALL_PREFIX}:${CMAKE_PREFIX_PATH:-}" + export LD_LIBRARY_PATH="${AWS_INSTALL_PREFIX}/lib:${AWS_INSTALL_PREFIX}/lib64:${LD_LIBRARY_PATH:-}" + CC="$cc_bin" CXX="$cxx_bin" run cmake .. + run make -j"$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" + run cmake --install . + ) + + note "Build completed." + note "Shared library: ${BUILD_DIR}/libpixels-index-rockset.so" + note "Installed JNI and RocksDB runtime libraries into $PIXELS_HOME/lib" +} + +main "$@" diff --git a/cpp/pixels-index/pixels-index-rockset/cmake/PatchRocksDBCloud.cmake b/cpp/pixels-index/pixels-index-rockset/cmake/PatchRocksDBCloud.cmake new file mode 100644 index 0000000000..066fce1429 --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/cmake/PatchRocksDBCloud.cmake @@ -0,0 +1,35 @@ +set(ROCKSDB_CLOUD_PATCH_DIR "${CMAKE_CURRENT_LIST_DIR}/patches") +set(ROCKSDB_CLOUD_PATCH_STAMP + "${ROCKSDB_CLOUD_SOURCE_DIR}/.pixels-rocksdb-cloud-patched") + +if(EXISTS "${ROCKSDB_CLOUD_PATCH_STAMP}") + message(STATUS "rocksdb-cloud patches already applied") + return() +endif() + +set(ROCKSDB_CLOUD_PATCH_FILES + "${ROCKSDB_CLOUD_PATCH_DIR}/0001-add-cloud-sources-and-aws-libs.patch" + "${ROCKSDB_CLOUD_PATCH_DIR}/0002-enable-path-style-s3-client.patch" + "${ROCKSDB_CLOUD_PATCH_DIR}/0003-include-cstdint-in-data-block-hash-index.patch") + +foreach(ROCKSDB_CLOUD_PATCH_FILE IN LISTS ROCKSDB_CLOUD_PATCH_FILES) + if(NOT EXISTS "${ROCKSDB_CLOUD_PATCH_FILE}") + message(FATAL_ERROR "rocksdb-cloud patch file not found: ${ROCKSDB_CLOUD_PATCH_FILE}") + endif() + + execute_process( + COMMAND patch --batch -p1 --input "${ROCKSDB_CLOUD_PATCH_FILE}" + WORKING_DIRECTORY "${ROCKSDB_CLOUD_SOURCE_DIR}" + RESULT_VARIABLE ROCKSDB_CLOUD_PATCH_RESULT + OUTPUT_VARIABLE ROCKSDB_CLOUD_PATCH_STDOUT + ERROR_VARIABLE ROCKSDB_CLOUD_PATCH_STDERR) + + if(NOT ROCKSDB_CLOUD_PATCH_RESULT EQUAL 0) + message(FATAL_ERROR + "Failed to apply rocksdb-cloud patch: ${ROCKSDB_CLOUD_PATCH_FILE}\n" + "stdout:\n${ROCKSDB_CLOUD_PATCH_STDOUT}\n" + "stderr:\n${ROCKSDB_CLOUD_PATCH_STDERR}") + endif() +endforeach() + +file(WRITE "${ROCKSDB_CLOUD_PATCH_STAMP}" "patched\n") diff --git a/cpp/pixels-index/pixels-index-rockset/cmake/RocksDBCloudCompat.h b/cpp/pixels-index/pixels-index-rockset/cmake/RocksDBCloudCompat.h new file mode 100644 index 0000000000..ab7d51de2b --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/cmake/RocksDBCloudCompat.h @@ -0,0 +1,4 @@ +#pragma once + +#include +#include diff --git a/cpp/pixels-index/pixels-index-rockset/cmake/patches/0001-add-cloud-sources-and-aws-libs.patch b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0001-add-cloud-sources-and-aws-libs.patch new file mode 100644 index 0000000000..59cc521061 --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0001-add-cloud-sources-and-aws-libs.patch @@ -0,0 +1,40 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 9db02a1e..5e18ba00 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -725,6 +725,21 @@ set(SOURCES + utilities/ttl/db_ttl_impl.cc + utilities/write_batch_with_index/write_batch_with_index.cc + utilities/write_batch_with_index/write_batch_with_index_internal.cc ++ cloud/aws/aws_env.cc ++ cloud/aws/aws_kafka.cc ++ cloud/aws/aws_kinesis.cc ++ cloud/aws/aws_retry.cc ++ cloud/aws/aws_s3.cc ++ cloud/db_cloud_impl.cc ++ cloud/cloud_env.cc ++ cloud/cloud_env_impl.cc ++ cloud/cloud_env_options.cc ++ cloud/cloud_log_controller.cc ++ cloud/manifest_reader.cc ++ cloud/purge.cc ++ cloud/cloud_manifest.cc ++ cloud/cloud_storage_provider.cc ++ db/db_impl/db_impl_remote_compaction.cc + $) + + if(HAVE_SSE42 AND NOT MSVC) +@@ -792,6 +807,13 @@ if(WITH_LIBRADOS) + list(APPEND THIRDPARTY_LIBS rados) + endif() + ++list(APPEND THIRDPARTY_LIBS ++ aws-cpp-sdk-s3 ++ aws-cpp-sdk-transfer ++ aws-cpp-sdk-kinesis ++ aws-cpp-sdk-core ++ curl) ++ + if(WIN32) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + else() diff --git a/cpp/pixels-index/pixels-index-rockset/cmake/patches/0002-enable-path-style-s3-client.patch b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0002-enable-path-style-s3-client.patch new file mode 100644 index 0000000000..bc446fb9c5 --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0002-enable-path-style-s3-client.patch @@ -0,0 +1,16 @@ +diff --git a/cloud/aws/aws_s3.cc b/cloud/aws/aws_s3.cc +index 5b3c457d..aa9370d3 100644 +--- a/cloud/aws/aws_s3.cc ++++ b/cloud/aws/aws_s3.cc +@@ -107,9 +107,9 @@ class AwsS3ClientWrapper { + const CloudEnvOptions& cloud_options) + : cloud_request_callback_(cloud_options.cloud_request_callback) { + if (creds) { +- client_ = std::make_shared(creds, config); ++ client_ = std::make_shared(creds, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent, true); + } else { +- client_ = std::make_shared(config); ++ client_ = std::make_shared(config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent, true); + } + if (cloud_options.use_aws_transfer_manager) { + Aws::Transfer::TransferManagerConfiguration transferManagerConfig( diff --git a/cpp/pixels-index/pixels-index-rockset/cmake/patches/0003-include-cstdint-in-data-block-hash-index.patch b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0003-include-cstdint-in-data-block-hash-index.patch new file mode 100644 index 0000000000..bad9b74b1c --- /dev/null +++ b/cpp/pixels-index/pixels-index-rockset/cmake/patches/0003-include-cstdint-in-data-block-hash-index.patch @@ -0,0 +1,12 @@ +diff --git a/table/block_based/data_block_hash_index.h b/table/block_based/data_block_hash_index.h +index 0af8b257..75a797bd 100644 +--- a/table/block_based/data_block_hash_index.h ++++ b/table/block_based/data_block_hash_index.h +@@ -5,6 +5,7 @@ + + #pragma once + ++#include + #include + #include + diff --git a/cpp/pixels-index/pixels-index-rockset/include/jni/io_pixelsdb_pixels_index_rockset_jni_RocksetDB.h b/cpp/pixels-index/pixels-index-rockset/include/jni/io_pixelsdb_pixels_index_rockset_jni_RocksetDB.h index 84816951f4..74f4d27586 100644 --- a/cpp/pixels-index/pixels-index-rockset/include/jni/io_pixelsdb_pixels_index_rockset_jni_RocksetDB.h +++ b/cpp/pixels-index/pixels-index-rockset/include/jni/io_pixelsdb_pixels_index_rockset_jni_RocksetDB.h @@ -26,10 +26,10 @@ JNIEXPORT jlongArray JNICALL Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB /* * Class: io_pixelsdb_pixels_index_rockset_jni_RocksetDB * Method: listColumnFamilies0 - * Signature: (Ljava/lang/String;)Ljava/util/List; + * Signature: (JLjava/lang/String;)Ljava/util/List; */ JNIEXPORT jobject JNICALL Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_listColumnFamilies0 - (JNIEnv *, jclass, jstring); + (JNIEnv *, jclass, jlong, jstring); /* * Class: io_pixelsdb_pixels_index_rockset_jni_RocksetDB diff --git a/cpp/pixels-index/pixels-index-rockset/include/jni/portal.h b/cpp/pixels-index/pixels-index-rockset/include/jni/portal.h index 855970d825..a153fc99f3 100644 --- a/cpp/pixels-index/pixels-index-rockset/include/jni/portal.h +++ b/cpp/pixels-index/pixels-index-rockset/include/jni/portal.h @@ -19,7 +19,6 @@ #include "rocksdb/rate_limiter.h" #include "rocksdb/status.h" #include "rocksdb/table.h" -#include "rocksdb/utilities/backup_engine.h" #include "rocksdb/utilities/memory_util.h" #include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/write_batch_with_index.h" @@ -27,6 +26,31 @@ namespace ROCKSDB_NAMESPACE { class JniUtil { public: + template + static std::vector fromJPointers(JNIEnv* env, jlongArray handles, + jboolean* has_exception) { + std::vector ptrs; + *has_exception = JNI_FALSE; + if (handles == nullptr) { + return ptrs; + } + + const jsize handle_count = env->GetArrayLength(handles); + jlong* raw_handles = env->GetLongArrayElements(handles, nullptr); + if (raw_handles == nullptr) { + *has_exception = JNI_TRUE; + return ptrs; + } + + ptrs.reserve(static_cast(handle_count)); + for (jsize i = 0; i < handle_count; ++i) { + ptrs.push_back(reinterpret_cast(raw_handles[i])); + } + + env->ReleaseLongArrayElements(handles, raw_handles, JNI_ABORT); + return ptrs; + } + /** * Detect if jlong overflows size_t * @@ -201,4 +225,82 @@ class MemoryUsageTypeJni { } } }; -} \ No newline at end of file + +class ByteJni { + public: + static jobject valueOf(JNIEnv* env, jbyte value) { + jclass cls = env->FindClass("java/lang/Byte"); + if (cls == nullptr) { + return nullptr; + } + jmethodID mid = env->GetStaticMethodID(cls, "valueOf", "(B)Ljava/lang/Byte;"); + if (mid == nullptr) { + return nullptr; + } + return env->CallStaticObjectMethod(cls, mid, value); + } +}; + +class LongJni { + public: + static jobject valueOf(JNIEnv* env, jlong value) { + jclass cls = env->FindClass("java/lang/Long"); + if (cls == nullptr) { + return nullptr; + } + jmethodID mid = env->GetStaticMethodID(cls, "valueOf", "(J)Ljava/lang/Long;"); + if (mid == nullptr) { + return nullptr; + } + return env->CallStaticObjectMethod(cls, mid, value); + } +}; + +class HashMapJni { + public: + template + using FnMapKV = + std::function>(const std::pair&)>; + + static jobject construct(JNIEnv* env, uint32_t initial_capacity) { + jclass cls = env->FindClass("java/util/HashMap"); + if (cls == nullptr) { + return nullptr; + } + jmethodID mid = env->GetMethodID(cls, "", "(I)V"); + if (mid == nullptr) { + return nullptr; + } + return env->NewObject(cls, mid, static_cast(initial_capacity)); + } + + template + static bool putAll(JNIEnv* env, jobject hash_map, Iterator begin, + Iterator end, Fn fn_map_kv) { + jclass cls = env->FindClass("java/util/HashMap"); + if (cls == nullptr) { + return false; + } + jmethodID mid = env->GetMethodID( + cls, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + if (mid == nullptr) { + return false; + } + + for (auto it = begin; it != end; ++it) { + auto kv = fn_map_kv(*it); + if (kv == nullptr) { + return false; + } + jobject old_value = env->CallObjectMethod(hash_map, mid, kv->first, kv->second); + if (env->ExceptionCheck()) { + return false; + } + env->DeleteLocalRef(old_value); + env->DeleteLocalRef(kv->first); + env->DeleteLocalRef(kv->second); + } + return true; + } +}; +} diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/BlockBasedTableConfig.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/BlockBasedTableConfig.cpp index b8ecbeba91..d07d83ae3a 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/BlockBasedTableConfig.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/BlockBasedTableConfig.cpp @@ -21,10 +21,9 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetBlockBasedTableConfig_newTableF JNIEnv*, jclass, jlong jblock_cache_handle, - jlong jfilter_policy_handle, jlong jblock_size, - jboolean jwhole_key_filtering, - jboolean jcache_index_and_filter_blocks) + jlong jfilter_policy_handle, + jboolean jwhole_key_filtering) { BlockBasedTableOptions options; // 1. block cache diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/ColumnFamilyOptions.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/ColumnFamilyOptions.cpp index 125bf8e3a5..04d873e83e 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/ColumnFamilyOptions.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/ColumnFamilyOptions.cpp @@ -1,6 +1,7 @@ #include "io_pixelsdb_pixels_index_rockset_jni_RocksetColumnFamilyOptions.h" #include #include +#include using ROCKSDB_NAMESPACE::ColumnFamilyOptions; using ROCKSDB_NAMESPACE::CompressionType; @@ -26,7 +27,9 @@ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_index_rockset_jni_RocksetColumnFamilyOptions_nativeSetWriteBufferSize( JNIEnv* env, jclass, jlong jhandle, jlong jwrite_buffer_size) { - if (jwrite_buffer_size < 0 || jwrite_buffer_size > static_cast(SIZE_MAX)) { + if (jwrite_buffer_size < 0 || + static_cast(jwrite_buffer_size) > + static_cast(std::numeric_limits::max())) { jclass exc = env->FindClass("java/lang/IllegalArgumentException"); env->ThrowNew(exc, "write_buffer_size out of range"); return; @@ -141,4 +144,3 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetColumnFamilyOptions_disposeInte } - diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/LRUCache.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/LRUCache.cpp index baef4d6c2f..644a8269b7 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/LRUCache.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/LRUCache.cpp @@ -31,8 +31,7 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetLRUCache_newLRUCache( static_cast(jhigh_pri_pool_ratio), nullptr /* memory_allocator */, ROCKSDB_NAMESPACE::kDefaultToAdaptiveMutex, - ROCKSDB_NAMESPACE::kDefaultCacheMetadataChargePolicy, - static_cast(jlow_pri_pool_ratio))); + ROCKSDB_NAMESPACE::kDefaultCacheMetadataChargePolicy)); return reinterpret_cast(sptr_lru_cache); } diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/ReadOptions.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/ReadOptions.cpp index da0a1d323e..0814d9a266 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/ReadOptions.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/ReadOptions.cpp @@ -44,8 +44,8 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetReadOptions_disposeInternalJni( */ JNIEXPORT void JNICALL Java_io_pixelsdb_pixels_index_rockset_jni_RocksetReadOptions_setPrefixSameAsStart( - JNIEnv*, jclass, jlong handle, jboolean flag) + JNIEnv*, jobject, jlong handle, jboolean flag) { reinterpret_cast(handle)->prefix_same_as_start = static_cast(flag); -} \ No newline at end of file +} diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/Rockset.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/Rockset.cpp index 78d659f523..d5a3d84581 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/Rockset.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/Rockset.cpp @@ -2,12 +2,25 @@ #include #include +#include "rocksdb/pluggable_compaction.h" #include "rocksdb/cloud/db_cloud.h" #include "rocksdb/options.h" #include "rocksdb/db.h" #include "portal.h" #include "cplusplus_to_java_convert.h" +namespace { + +void ThrowJavaException(JNIEnv* env, const char* class_name, + const std::string& message) { + jclass exception_class = env->FindClass(class_name); + if (exception_class != nullptr) { + env->ThrowNew(exception_class, message.c_str()); + } +} + +} // namespace + /** * This file is modified from RocksDB's own JNI bindings. * @author Rolland1944 @@ -24,11 +37,15 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_open( jobjectArray jcf_names, jlongArray jcf_options_handles) { - // 1. Options* - auto* options = - reinterpret_cast(joptions); - assert(options != nullptr); - options->env = reinterpret_cast(cloud_env_ptr); + // 1. The Java RocksetDBOptions handle owns a rocksdb::DBOptions, while + // DBCloud::Open requires rocksdb::Options. + auto* db_options = + reinterpret_cast(joptions); + assert(db_options != nullptr); + + ROCKSDB_NAMESPACE::Options options; + static_cast(options) = *db_options; + options.env = reinterpret_cast(cloud_env_ptr); // 2. db path const char* db_path_chars = @@ -74,7 +91,7 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_open( ROCKSDB_NAMESPACE::DBCloud* db = nullptr; auto status = ROCKSDB_NAMESPACE::DBCloud::Open( - *options, + options, db_path, cf_descs, "" /* persistent_cache_path */, @@ -85,6 +102,9 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_open( ); if (!status.ok()) { + ThrowJavaException(env, "java/io/IOException", + "Failed to open RocksDB Cloud database at " + db_path + + ": " + status.ToString()); return nullptr; } @@ -121,7 +141,7 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_closeDatabase( JNIEXPORT jobject JNICALL Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_listColumnFamilies0( - JNIEnv* env, jclass, jstring jdb_path) + JNIEnv* env, jclass, jlong cloud_env_ptr, jstring jdb_path) { const char* path_chars = env->GetStringUTFChars(jdb_path, nullptr); if (path_chars == nullptr) { @@ -129,11 +149,28 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_listColumnFamilies0( } std::string db_path(path_chars); env->ReleaseStringUTFChars(jdb_path, path_chars); + + auto* cloud_env = + reinterpret_cast(cloud_env_ptr); + if (cloud_env == nullptr) { + ThrowJavaException(env, "java/lang/IllegalArgumentException", + "CloudEnv handle is null"); + return nullptr; + } + + ROCKSDB_NAMESPACE::Status st = cloud_env->PreloadCloudManifest(db_path); + if (!st.ok()) { + ThrowJavaException(env, "java/io/IOException", + "Failed to preload RocksDB Cloud manifest at " + + db_path + ": " + st.ToString()); + return nullptr; + } + ROCKSDB_NAMESPACE::Options options; + options.env = cloud_env; std::vector column_families; - ROCKSDB_NAMESPACE::Status st = - ROCKSDB_NAMESPACE::DBCloud::ListColumnFamilies( - options, db_path, &column_families); + st = ROCKSDB_NAMESPACE::DBCloud::ListColumnFamilies( + options, db_path, &column_families); if (!st.ok()) { jclass ex = env->FindClass("java/lang/RuntimeException"); @@ -164,8 +201,13 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_listColumnFamilies0( } JNIEXPORT jlong JNICALL -Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_createColumnFamily0( - JNIEnv* env, jobject jdb, jlong jhandle, jbyteArray jcf_name) +Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_createColumnFamily( + JNIEnv* env, + jclass, + jlong jhandle, + jbyteArray jcf_name, + jint jcf_name_len, + jlong jcf_options) { auto* db = reinterpret_cast(jhandle); if (db == nullptr) { @@ -175,17 +217,29 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_createColumnFamily0( } jsize len = env->GetArrayLength(jcf_name); + if (jcf_name_len != len) { + jclass ex = env->FindClass("java/lang/IllegalArgumentException"); + env->ThrowNew(ex, "Column family name length does not match byte array length"); + return reinterpret_cast(nullptr); + } + std::string cf_name; cf_name.resize(len); env->GetByteArrayRegion( jcf_name, 0, len, reinterpret_cast(&cf_name[0])); - ROCKSDB_NAMESPACE::ColumnFamilyOptions cf_options; + auto* cf_options = + reinterpret_cast(jcf_options); + if (cf_options == nullptr) { + jclass ex = env->FindClass("java/lang/IllegalArgumentException"); + env->ThrowNew(ex, "Column family options handle is null"); + return reinterpret_cast(nullptr); + } ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle = nullptr; ROCKSDB_NAMESPACE::Status st = - db->CreateColumnFamily(cf_options, cf_name, &cf_handle); + db->CreateColumnFamily(*cf_options, cf_name, &cf_handle); if (!st.ok()) { jclass ex = env->FindClass("java/lang/RuntimeException"); @@ -193,16 +247,6 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_createColumnFamily0( return reinterpret_cast(nullptr); } - auto* sptr = - new std::shared_ptr(cf_handle); - - jclass cf_handle_clz = - env->FindClass( - "io/pixelsdb/pixels/index/rockset/jni/RocksetColumnFamilyHandle"); - - jmethodID ctor = - env->GetMethodID(cf_handle_clz, "", "(J)V"); - return GET_CPLUSPLUS_POINTER(cf_handle); } @@ -350,4 +394,3 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetDB_iterator( auto* it = db->NewIterator(*ro, cf); return reinterpret_cast(it); } - diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/RocksetEnv.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/RocksetEnv.cpp index 0a897ca0f1..e1021a4350 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/RocksetEnv.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/RocksetEnv.cpp @@ -1,7 +1,6 @@ #include "io_pixelsdb_pixels_index_rockset_jni_RocksetEnv.h" +#include "rocksdb/pluggable_compaction.h" #include "rocksdb/cloud/db_cloud.h" -#include "rocksdb/cloud/cloud_file_system.h" -#include /** * This file is modified from RocksDB's own JNI bindings. @@ -27,20 +26,17 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetEnv_createCloudFileSystem0( return 0; } - // 2. AWS SDK - Aws::SDKOptions aws_options; - Aws::InitAPI(aws_options); - - // 3. CloudFileSystemOptions - ROCKSDB_NAMESPACE::CloudFileSystemOptions cfs_options; - cfs_options.credentials.InitializeSimple( + // 2. CloudEnvOptions + ROCKSDB_NAMESPACE::CloudEnvOptions cloud_env_options; + cloud_env_options.credentials.InitializeSimple( getenv("AWS_ACCESS_KEY_ID"), getenv("AWS_SECRET_ACCESS_KEY")); - if (!cfs_options.credentials.HasValid().ok()) { + auto cred_status = cloud_env_options.credentials.HasValid(); + if (!cred_status.ok()) { env->ThrowNew( env->FindClass("java/lang/SecurityException"), - "Invalid AWS credentials"); + cred_status.ToString().c_str()); return 0; } @@ -50,26 +46,30 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetEnv_createCloudFileSystem0( const char* prefix_chars = env->GetStringUTFChars(js3_prefix, nullptr); - cfs_options.src_bucket.SetBucketName(bucket_chars); - cfs_options.src_bucket.SetObjectPath(prefix_chars); - cfs_options.src_bucket.SetRegion( - getenv("AWS_DEFAULT_REGION")); + const std::string bucket_name(bucket_chars); + const std::string object_prefix(prefix_chars); + const std::string region(getenv("AWS_DEFAULT_REGION")); env->ReleaseStringUTFChars(jbucket_name, bucket_chars); env->ReleaseStringUTFChars(js3_prefix, prefix_chars); - // 5. base Env + // 4. base Env ROCKSDB_NAMESPACE::Env* base_env = ROCKSDB_NAMESPACE::Env::Default(); - // 6. Create CloudFileSystem - std::shared_ptr base_fs = - base_env->GetFileSystem(); + cloud_env_options.src_bucket.SetBucketName(bucket_name, ""); + cloud_env_options.src_bucket.SetObjectPath(object_prefix); + cloud_env_options.src_bucket.SetRegion(region); + cloud_env_options.dest_bucket.SetBucketName(bucket_name, ""); + cloud_env_options.dest_bucket.SetObjectPath(object_prefix); + cloud_env_options.dest_bucket.SetRegion(region); - ROCKSDB_NAMESPACE::CloudFileSystem* cloud_fs = nullptr; + // 5. Create CloudEnv + ROCKSDB_NAMESPACE::CloudEnv* cloud_env = nullptr; auto status = - ROCKSDB_NAMESPACE::CloudFileSystemEnv::NewAwsFileSystem( - base_fs, cfs_options, nullptr, &cloud_fs); + ROCKSDB_NAMESPACE::CloudEnv::NewAwsEnv( + base_env, + cloud_env_options, nullptr, &cloud_env); if (!status.ok()) { env->ThrowNew( @@ -78,13 +78,7 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetEnv_createCloudFileSystem0( return 0; } - // 7. Composite Env - std::shared_ptr fs(cloud_fs); - std::unique_ptr cloud_env = - ROCKSDB_NAMESPACE::CloudFileSystemEnv::NewCompositeEnv( - base_env, std::move(fs)); - - return reinterpret_cast(cloud_env.release()); + return reinterpret_cast(cloud_env); } JNIEXPORT void JNICALL @@ -102,4 +96,3 @@ Java_io_pixelsdb_pixels_index_rockset_jni_RocksetEnv_disposeInternalJni( delete env; } - diff --git a/cpp/pixels-index/pixels-index-rockset/lib/jni/memory_util.cpp b/cpp/pixels-index/pixels-index-rockset/lib/jni/memory_util.cpp index da0dae66f4..bc87c58337 100644 --- a/cpp/pixels-index/pixels-index-rockset/lib/jni/memory_util.cpp +++ b/cpp/pixels-index/pixels-index-rockset/lib/jni/memory_util.cpp @@ -7,7 +7,6 @@ #include #include -#include "include/org_rocksdb_MemoryUtil.h" #include "portal.h" jobject Java_io_pixelsdb_pixels_index_rockset_jni_MemoryUtil_getApproximateMemoryUsageByType( diff --git a/install.sh b/install.sh index 7b5d9be35c..fb6c67aef6 100755 --- a/install.sh +++ b/install.sh @@ -29,6 +29,15 @@ mkdir -p $PIXELS_HOME/listener mkdir -p $PIXELS_HOME/logs mkdir -p $PIXELS_HOME/var +echo "Installing pixels-index-rockset JNI library..." +ROCKSET_JNI_LIB=./cpp/pixels-index/pixels-index-rockset/build/libpixels-index-rockset.so +if [ ! -f "$ROCKSET_JNI_LIB" ]; then + echo "WARN: '$ROCKSET_JNI_LIB' not found. Skip installing pixels-index-rockset JNI library." + echo "WARN: Build it manually if you need Rockset index support. See cpp/pixels-index/pixels-index-rockset/README.md." +else + cp -v "$ROCKSET_JNI_LIB" $PIXELS_HOME/lib +fi + echo "Installing scripts..." CP_SBIN=0 @@ -166,4 +175,3 @@ echo "$( tput setaf 1 tput setab 7 )See the README of pixels-presto/trino/hive to install a query engine.$(tput sgr 0)" - diff --git a/pixels-index/pixels-index-rockset/pom.xml b/pixels-index/pixels-index-rockset/pom.xml index eb713e3bf6..07b6a7c1df 100644 --- a/pixels-index/pixels-index-rockset/pom.xml +++ b/pixels-index/pixels-index-rockset/pom.xml @@ -44,6 +44,21 @@ grpc-testing test + + io.grpc + grpc-netty-shaded + test + + + io.grpc + grpc-protobuf + test + + + io.grpc + grpc-stub + test + diff --git a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetFactory.java b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetFactory.java index 6aa3c0f564..4ec4afd697 100644 --- a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetFactory.java +++ b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetFactory.java @@ -77,11 +77,13 @@ private static RocksetEnv createRocksetEnv() } static RocksetDB createRocksetDB(String rocksetPath) throws Exception { + RocksetEnv rocksetEnv = createRocksetEnv(); + // 1. Get existing column families (returns empty list for new database) List existingColumnFamilies; try { - existingColumnFamilies = RocksetDB.listColumnFamilies0(rocksetPath); + existingColumnFamilies = RocksetDB.listColumnFamilies0(rocksetEnv.nativeHandle(), rocksetPath); } catch (Exception e) { // For new database, return list containing only default column family @@ -112,21 +114,7 @@ static RocksetDB createRocksetDB(String rocksetPath) throws Exception List descriptors = new ArrayList<>(); for(byte[] existingColumnFamily: existingColumnFamilies) { - long[] ids = IndexUtils.parseTableAndIndexId(existingColumnFamily); - Integer keyLen = null; - if(ids != null) - { - long tableId = ids[0]; - long indexId = ids[1]; - try - { - keyLen = getIndexKeyLen(tableId, indexId); - } catch (MetadataException ignored) - { - - } - } - descriptors.add(createCFDescriptor(existingColumnFamily, keyLen)); + descriptors.add(createCFDescriptor(existingColumnFamily)); } // 4. Open DB List handles = new ArrayList<>(); @@ -163,8 +151,7 @@ static RocksetDB createRocksetDB(String rocksetPath) throws Exception .setStatsDumpPeriodSec(statsInterval) .setDbLogDir(statsPath); } - RocksetEnv rocksetEnv = createRocksetEnv(); - RocksetDB db = RocksetDB.open(rocksetEnv, options, rocksetPath, descriptors, handles); + RocksetDB db = openWithMissingColumnFamilyRetry(rocksetEnv, options, rocksetPath, descriptors, handles); if(enableStats) { startRocksetLogThread(db); @@ -179,6 +166,103 @@ static RocksetDB createRocksetDB(String rocksetPath) throws Exception return db; } + private static RocksetDB openWithMissingColumnFamilyRetry( + RocksetEnv rocksetEnv, + RocksetDBOptions options, + String rocksetPath, + List descriptors, + List handles) throws Exception + { + Set descriptorNames = new HashSet<>(); + for (RocksetColumnFamilyDescriptor descriptor : descriptors) + { + descriptorNames.add(new String(descriptor.getName(), StandardCharsets.UTF_8)); + } + + Exception lastException = null; + for (int attempt = 0; attempt < 4; attempt++) + { + handles.clear(); + try + { + return RocksetDB.open(rocksetEnv, options, rocksetPath, descriptors, handles); + } catch (Exception e) + { + lastException = e; + List missingColumnFamilies = extractMissingColumnFamilies(e); + if (missingColumnFamilies.isEmpty()) + { + throw e; + } + + boolean added = false; + for (String columnFamily : missingColumnFamilies) + { + if (descriptorNames.add(columnFamily)) + { + descriptors.add(createCFDescriptor(columnFamily.getBytes(StandardCharsets.UTF_8))); + added = true; + } + } + if (!added) + { + throw e; + } + } + } + throw lastException; + } + + private static List extractMissingColumnFamilies(Throwable throwable) + { + final String marker = "Column families not opened:"; + for (Throwable current = throwable; current != null; current = current.getCause()) + { + String message = current.getMessage(); + if (message == null) + { + continue; + } + int markerIndex = message.indexOf(marker); + if (markerIndex < 0) + { + continue; + } + + String names = message.substring(markerIndex + marker.length()).trim(); + List result = new ArrayList<>(); + for (String name : names.split(",")) + { + String trimmed = name.trim(); + if (!trimmed.isEmpty()) + { + result.add(trimmed); + } + } + return result; + } + return Collections.emptyList(); + } + + private static RocksetColumnFamilyDescriptor createCFDescriptor(byte[] name) throws Exception + { + long[] ids = IndexUtils.parseTableAndIndexId(name); + Integer keyLen = null; + if(ids != null) + { + long tableId = ids[0]; + long indexId = ids[1]; + try + { + keyLen = getIndexKeyLen(tableId, indexId); + } catch (MetadataException ignored) + { + + } + } + return createCFDescriptor(name, keyLen); + } + private static RocksetColumnFamilyDescriptor createCFDescriptor(byte[] name, Integer keyLen) { ConfigFactory config = ConfigFactory.Instance(); diff --git a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetIndex.java b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetIndex.java index 5db5aff273..22e88d5934 100644 --- a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetIndex.java +++ b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/RocksetIndex.java @@ -49,6 +49,7 @@ public class RocksetIndex extends CachingSinglePointIndex { + private static final Logger LOGGER = LogManager.getLogger(RocksetIndex.class); // load pixels-index-rockset static { @@ -69,9 +70,8 @@ public class RocksetIndex extends CachingSinglePointIndex throw new IllegalStateException("libpixels-index-rockset.so is not readable at " + libPath); } System.load(libPath); - System.out.println("JNI loaded OK"); + LOGGER.info("RockSet Index JNI loaded OK"); } - private static final Logger LOGGER = LogManager.getLogger(RocksetIndex.class); private static final long TOMBSTONE_ROW_ID = Long.MAX_VALUE; private final RocksetDB rocksetDB; private final String rocksDBPath; diff --git a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetDB.java b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetDB.java index cf7d6912ac..58aa34bc48 100644 --- a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetDB.java +++ b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetDB.java @@ -66,6 +66,10 @@ public static RocksetDB open(RocksetEnv env, RocksetDBOptions var0, String var1, } else { long[] var11 = open(env.nativeHandle(), var0.nativeHandle, var1, var4, var5); + if (var11 == null) + { + throw new IllegalStateException("Native RocksDB Cloud open returned null without throwing an exception"); + } RocksetDB var12 = new RocksetDB(var11[0]); var12.storeOptionsInstance(var0); @@ -84,7 +88,7 @@ public static RocksetDB open(RocksetEnv env, RocksetDBOptions var0, String var1, public void put(RocksetColumnFamilyHandle var1, RocksetWriteOptions var2, ByteBuffer var3, ByteBuffer var4) throws RuntimeException { if (var3.isDirect() && var4.isDirect()) { - putDirect(this.nativeHandle, var2.nativeHandle, var3, var3.position(), var3.remaining(), var4, var4.position(), var4.remaining(), var1.nativeHandle); + putDirect(this.nativeHandle, var1.nativeHandle, var3, var3.position(), var3.remaining(), var4, var4.position(), var4.remaining(), var2.nativeHandle); } else { @@ -97,7 +101,7 @@ public void put(RocksetColumnFamilyHandle var1, RocksetWriteOptions var2, ByteBu assert var4.hasArray(); - put(this.nativeHandle, var2.nativeHandle, var3.array(), var3.arrayOffset() + var3.position(), var3.remaining(), var4.array(), var4.arrayOffset() + var4.position(), var4.remaining(), var1.nativeHandle); + put(this.nativeHandle, var1.nativeHandle, var3.array(), var3.arrayOffset() + var3.position(), var3.remaining(), var4.array(), var4.arrayOffset() + var4.position(), var4.remaining(), var2.nativeHandle); } var3.position(var3.limit()); @@ -185,7 +189,7 @@ public long getLongProperty( private static native void closeDatabase(long var0); private static native long[] open(long env_handle, long options_handle, String db_path, byte[][] descriptors, long[] cf_handles); - public static native List listColumnFamilies0(String dbPath); + public static native List listColumnFamilies0(long envHandle, String dbPath); private static native long createColumnFamily(final long handle, final byte[] columnFamilyName, final int columnFamilyNamelen, final long columnFamilyOptions) throws Exception; private static native void putDirect(long var0, long var2, ByteBuffer var4, int var5, int var6, ByteBuffer var7, int var8, int var9, long var10) throws RuntimeException; @@ -195,4 +199,3 @@ private static native long createColumnFamily(final long handle, final byte[] co private static native long getLongProperty(final long nativeHandle, final long cfHandle, final String property, final int propertyLength) throws Exception; } - diff --git a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetEnv.java b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetEnv.java index 1c87a3b744..c299f0fa6c 100644 --- a/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetEnv.java +++ b/pixels-index/pixels-index-rockset/src/main/java/io/pixelsdb/pixels/index/rockset/jni/RocksetEnv.java @@ -21,7 +21,7 @@ public static RocksetEnv create(String bucket, String prefix) return new RocksetEnv(h); } - long nativeHandle() + public long nativeHandle() { return nativeHandle; } diff --git a/pixels-index/pixels-index-rockset/src/test/java/io/pixelsdb/pixels/index/rockset/TestRocksetIndex.java b/pixels-index/pixels-index-rockset/src/test/java/io/pixelsdb/pixels/index/rockset/TestRocksetIndex.java index a6e2215aff..bec5b82114 100644 --- a/pixels-index/pixels-index-rockset/src/test/java/io/pixelsdb/pixels/index/rockset/TestRocksetIndex.java +++ b/pixels-index/pixels-index-rockset/src/test/java/io/pixelsdb/pixels/index/rockset/TestRocksetIndex.java @@ -103,49 +103,49 @@ public void testPutEntry() throws SinglePointIndexException assertEquals(rowId, storedRowId); } - @Test - public void testPutEntries() throws SinglePointIndexException, MainIndexException - { - long timestamp = 1000L; - long fileId = 1L; - int rgId = 2; - - List entries = new ArrayList<>(); - - // Create two entries - for (int i = 0; i < 2; i++) - { - byte[] key = ("testPutEntries" + i).getBytes(); // use different keys - - long rowId = i * 1000L; - - IndexProto.IndexKey keyProto = IndexProto.IndexKey.newBuilder() - .setIndexId(INDEX_ID).setKey(ByteString.copyFrom(key)).setTimestamp(timestamp).build(); - - IndexProto.RowLocation rowLocation = IndexProto.RowLocation.newBuilder() - .setFileId(fileId).setRgId(rgId).setRgRowOffset(i).build(); - - IndexProto.PrimaryIndexEntry entry = IndexProto.PrimaryIndexEntry.newBuilder() - .setIndexKey(keyProto).setRowId(rowId).setRowLocation(rowLocation).build(); - entries.add(entry); - } - - boolean success = uniqueIndex.putPrimaryEntries(entries); - assertTrue(success, "putEntries should return true"); - - // Assert every index has been written to rockset - for (int i = 0; i < entries.size(); i++) - { - IndexProto.PrimaryIndexEntry entry = entries.get(i); - ByteBuffer keyBuffer = toKeyBuffer(entry.getIndexKey()); - ByteBuffer valueBuffer = RocksetThreadResources.getValueBuffer(); - RocksetReadOptions rocksetReadOptions = RocksetThreadResources.getReadOptions(); - // int ret = rocksetDB.get(rocksetReadOptions, keyBuffer, valueBuffer); - // assertTrue(ret != 0); - long storedRowId = valueBuffer.getLong(); - assertEquals(i* 1000L, storedRowId); - } - } + // @Test + // public void testPutEntries() throws SinglePointIndexException, MainIndexException + // { + // long timestamp = 1000L; + // long fileId = 1L; + // int rgId = 2; + + // List entries = new ArrayList<>(); + + // // Create two entries + // for (int i = 0; i < 2; i++) + // { + // byte[] key = ("testPutEntries" + i).getBytes(); // use different keys + + // long rowId = i * 1000L; + + // IndexProto.IndexKey keyProto = IndexProto.IndexKey.newBuilder() + // .setIndexId(INDEX_ID).setKey(ByteString.copyFrom(key)).setTimestamp(timestamp).build(); + + // IndexProto.RowLocation rowLocation = IndexProto.RowLocation.newBuilder() + // .setFileId(fileId).setRgId(rgId).setRgRowOffset(i).build(); + + // IndexProto.PrimaryIndexEntry entry = IndexProto.PrimaryIndexEntry.newBuilder() + // .setIndexKey(keyProto).setRowId(rowId).setRowLocation(rowLocation).build(); + // entries.add(entry); + // } + + // boolean success = uniqueIndex.putPrimaryEntries(entries); + // assertTrue(success, "putEntries should return true"); + + // // Assert every index has been written to rockset + // for (int i = 0; i < entries.size(); i++) + // { + // IndexProto.PrimaryIndexEntry entry = entries.get(i); + // ByteBuffer keyBuffer = toKeyBuffer(entry.getIndexKey()); + // ByteBuffer valueBuffer = RocksetThreadResources.getValueBuffer(); + // RocksetReadOptions rocksetReadOptions = RocksetThreadResources.getReadOptions(); + // // int ret = rocksetDB.get(rocksetReadOptions, keyBuffer, valueBuffer); + // // assertTrue(ret != 0); + // long storedRowId = valueBuffer.getLong(); + // assertEquals(i* 1000L, storedRowId); + // } + // } @Test public void testGetUniqueRowId() throws SinglePointIndexException