[pull] memtable_as_log_index from topling:memtable_as_log_index

.github/workflows/android-jni.yml

@@ -0,0 +1,104 @@
+name: "build android-jni"
+
+on:
+  workflow_dispatch:
+    inputs:
+      version_suffix:
+        required: false
+        default: '-SNAPSHOT'
+      use_lto:
+        required: false
+        description: Link Use LTO
+        default: '1'
+      verbose_build_output:
+        required: false
+        description: run make with V=...
+        default: '0'
+      publish:
+        required: false
+        type: boolean
+        description: publish to github
+        default: true
+
+env:
+  NDK_VERSION: 26.1.10909125
+  ANDROID_ABI: arm64-v8a
+  ANDROID_PLATFORM: android-34
+  ANDROID_NDK_ROOT: ${{github.workspace}}/android-ndk
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    env:
+      GITHUB_TOKEN: ${{github.token}}
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: maven
+          server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+          settings-path: ${{github.workspace}} # location for the settings.xml file
+
+      - name: Setup Android SDK
+        uses: android-actions/setup-android@v3
+
+      - name: Display NDK information
+        run: |
+          set -xe
+          env | egrep 'ANDROID|NDK'
+          #find /usr/local/lib/android/sdk -type d
+          $ANDROID_NDK_HOME/ndk-build --version
+
+      - name: Init submodule & Setup ssh
+        run: |
+         git submodule update --init --recursive
+         mkdir -p ~/.ssh && mkdir -p /opt/lib
+         ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+         echo nproc = `nproc`
+
+      - name: Git clone SidePlugin(s)
+        run: |
+          export ROCKSDB_DISABLE_GFLAGS=1
+          make clean # this triggers auto git clone
+
+      - name: Compile RocksDB Java Release
+        run: |
+          set -xe
+          echo $JAVA_HOME
+          export V=${{inputs.verbose_build_output}}
+          export USE_LTO=${{inputs.use_lto}}
+          bash -x android-build.sh
+
+      - name: Generate java/pom.xml from template
+        run: |
+          set -xe
+          ROCKSDB_VERSION=`build_tools/version.sh full`
+          ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}}
+          cd java
+          # F\?ROCKSDB_JAVA_VERSION for both flink and normal rocksdbjni
+          sed -e 's/\${F\?ROCKSDB_JAVA_VERSION}/'"${ROCKSDB_JAVA_VERSION}/" \
+              -e 's/\(.*<artifactId>\)\(f\?rocksdbjni\)\(<\/artifactId>.*\)/\1\2-'"$ANDROID_PLATFORM"'\3/' \
+              pom.xml.template > pom.xml
+
+      - name: Publish rocksjni-android jar to GitHub Packages
+        run: |
+          ROCKSDB_VERSION=`build_tools/version.sh full`
+          ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}${{inputs.version_suffix}}
+          set -xe
+          cd java/target
+          mvn deploy:deploy-file -e -s $GITHUB_WORKSPACE/settings.xml \
+              -DpomFile=../pom.xml \
+              -Durl=https://maven.pkg.github.com/$GITHUB_REPOSITORY -DrepositoryId=github \
+              -Dfile=rocksdbjni-${ROCKSDB_VERSION}-linux64.jar \
+              -Dversion=${ROCKSDB_JAVA_VERSION}

.github/workflows/db_bench-avx512.yml

@@ -0,0 +1,93 @@
+# TODO: How to cache make files / speed up build progress here?
+name: "db_bench-avx512"
+
+on:
+  workflow_dispatch:
+    inputs:
+      use_clang:
+        type: boolean
+        required: false
+        default: false
+        description: use clang
+
+jobs:
+  build:
+    # refer https://github.com/actions/runner-images to get the details
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{github.token}}
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install Prerequisites
+        run: |
+         set -xe
+         sudo apt-get update -y && sudo apt-get install -y \
+         libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev \
+         libbz2-dev libcurl4-openssl-dev liburing-dev \
+         libsnappy-dev libbz2-dev liblz4-dev libzstd-dev
+
+      - name: Init submodule & Setup ssh
+        run: |
+         git submodule update --init --recursive
+         mkdir -p ~/.ssh && mkdir -p /opt/lib
+         ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+         echo nproc = `nproc`
+
+      - name: Git clone SidePlugin(s)
+        run: |
+          # this will be auto triggerred in normal build, we defined
+          # it here is just for seperate steps more clearly.
+          # there is no dedicated target for this purpose, so we use `clean`
+          # as the target, it need not any `clean` at this point.
+          # `clean` is just used for triggers auto git clone
+          make clean
+
+      - name: Compile db_bench
+        run: |
+          set -xe
+          if ${{inputs.use_clang}}; then
+            export CXX=clang++
+            export CC=clang
+          fi
+          export CPU=-march=skylake-avx512
+          make db_bench memtablerep_bench -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0
+
+      - name: Run db_bench
+        run: |
+          set -xe
+          make db_bench -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0
+          export TOPLINGDB_GetContext_sampling=kNone
+          export ROCKSDB_KICK_OUT_OPTIONS_FILE=1
+          args=(
+              -json sideplugin/rockside/sample-conf/db_bench_enterprise.yaml
+              -num=50000000
+              -key_size=8
+              -value_size=10
+              -batch_size=1000
+              -benchmarks=fillseq,compact,nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq,readrandom,readrandom,readrandom
+              -scan_omit_key -scan_omit_value # ToplingDB specific
+              -enable_zero_copy # ToplingDB specific, for point search by Get/MultiGet
+              -progress_reports=false
+          )
+          ./db_bench ${args[@]}
+
+      - name: Run memtablerep_bench
+        run: |
+          args=(
+            -benchmarks=fillrandom,readrandom,readwrite
+            -item_size=0
+            -num_operations=100000000
+            -write_buffer_size=536870912
+          )
+          set -xe
+          ./memtablerep_bench ${args[@]} -memtablerep=skiplist
+          echo
+          ./memtablerep_bench ${args[@]} -memtablerep='cspp:{"mem_cap":"16G","use_hugepage":false}'

.github/workflows/db_bench.yml

@@ -0,0 +1,92 @@
+# TODO: How to cache make files / speed up build progress here?
+name: "db_bench"
+
+on:
+  workflow_dispatch:
+    inputs:
+      use_clang:
+        type: boolean
+        required: false
+        default: false
+        description: use clang
+
+jobs:
+  build:
+    # refer https://github.com/actions/runner-images to get the details
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{github.token}}
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install Prerequisites
+        run: |
+         set -xe
+         sudo apt-get update -y && sudo apt-get install -y \
+         libjemalloc-dev libaio-dev libgflags-dev zlib1g-dev \
+         libbz2-dev libcurl4-openssl-dev liburing-dev \
+         libsnappy-dev libbz2-dev liblz4-dev libzstd-dev
+
+      - name: Init submodule & Setup ssh
+        run: |
+         git submodule update --init --recursive
+         mkdir -p ~/.ssh && mkdir -p /opt/lib
+         ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+         echo nproc = `nproc`
+
+      - name: Git clone SidePlugin(s)
+        run: |
+          # this will be auto triggerred in normal build, we defined
+          # it here is just for seperate steps more clearly.
+          # there is no dedicated target for this purpose, so we use `clean`
+          # as the target, it need not any `clean` at this point.
+          # `clean` is just used for triggers auto git clone
+          make clean
+
+      - name: Compile db_bench
+        run: |
+          set -xe
+          if ${{inputs.use_clang}}; then
+            export CXX=clang++
+            export CC=clang
+          fi
+          make db_bench memtablerep_bench -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0
+
+      - name: Run db_bench
+        run: |
+          set -xe
+          make db_bench -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0
+          export TOPLINGDB_GetContext_sampling=kNone
+          export ROCKSDB_KICK_OUT_OPTIONS_FILE=1
+          args=(
+              -json sideplugin/rockside/sample-conf/db_bench_enterprise.yaml
+              -num=50000000
+              -key_size=8
+              -value_size=10
+              -batch_size=1000
+              -benchmarks=fillseq,compact,nextwithkey,nextwithkey,nextwithkey,readseq,readseq,readseq,readrandom,readrandom,readrandom
+              -scan_omit_key -scan_omit_value # ToplingDB specific
+              -enable_zero_copy # ToplingDB specific, for point search by Get/MultiGet
+              -progress_reports=false
+          )
+          ./db_bench ${args[@]}
+
+      - name: Run memtablerep_bench
+        run: |
+          args=(
+            -benchmarks=fillrandom,readrandom,readwrite
+            -item_size=0
+            -num_operations=100000000
+            -write_buffer_size=536870912
+          )
+          set -xe
+          ./memtablerep_bench ${args[@]} -memtablerep=skiplist
+          echo
+          ./memtablerep_bench ${args[@]} -memtablerep='cspp:{"mem_cap":"16G","use_hugepage":false}'

.github/workflows/topling-jni-release.yml

@@ -0,0 +1,106 @@
+name: Publish to Release
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+          server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+          settings-path: ${{ github.workspace }} # location for the settings.xml file
+
+      - name: Install Prerequisites
+        run: |
+         set -xe
+         cat $GITHUB_WORKSPACE/settings.xml
+         sudo apt-get update -y && sudo apt-get install -y \
+         libaio-dev zlib1g-dev libcurl4-openssl-dev liburing-dev
+
+      - name: Init submodule & Setup ssh
+        run: |
+         git submodule update --init --recursive
+         mkdir -p ~/.ssh && mkdir -p /opt/lib
+         ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+         echo nproc = `nproc`
+
+      - name: Git clone SidePlugin(s)
+        run: |
+          # this will be auto triggerred in normal build, we defined
+          # it here is just for seperate steps more clearly.
+          # there is no dedicated target for this purpose, so we use `clean`
+          # as the target, it need not any `clean` at this point.
+          # `clean` is just used for triggers auto git clone
+          env USE_LTO=1 \
+              ROCKSDB_DISABLE_GFLAGS=1 \
+          make clean
+
+      - name: make bundled compression libs
+        run: |
+          env USE_LTO=1 \
+              ROCKSDB_DISABLE_GFLAGS=1 \
+          make UPDATE_REPO=0 DEBUG_LEVEL=0 DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 -j60 libsnappy.a liblz4.a libbz2.a
+
+      - name: make depend for Compile RocksDBJava Release
+        run: |
+          echo JAVA_HOME = $JAVA_HOME
+          set -xe
+          env USE_LTO=1 \
+              ROCKSDB_DISABLE_GFLAGS=1 \
+          make depend java/include/java_header_list.mk \
+               -j`nproc` DEBUG_LEVEL=0 UPDATE_REPO=0 \
+               DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 \
+               STRIP_DEBUG_INFO=1 ROCKSDB_JAR_WITH_DYNAMIC_LIBS=1
+
+      - name: Compile RocksDBJava Release
+        run: |
+          echo JAVA_HOME = $JAVA_HOME
+          set -xe
+          env USE_LTO=1 \
+              ROCKSDB_DISABLE_GFLAGS=1 \
+          make rocksdbjava install-dcompact -j`nproc` \
+               DEBUG_LEVEL=0 UPDATE_REPO=0 PREFIX=install-here \
+               DISABLE_JEMALLOC=1 TOPLING_USE_DYNAMIC_TLS=1 \
+               STRIP_DEBUG_INFO=1 ROCKSDB_JAR_WITH_DYNAMIC_LIBS=1
+
+      - name: Strip and Change dcompact_worker
+        run: |
+          MAJOR_DOT_MINOR=`build_tools/version.sh major`.`build_tools/version.sh minor`
+          exebin=install-here/bin/dcompact_worker.exe
+          strip ${exebin}
+          patchelf --replace-needed librocksdb.so.${MAJOR_DOT_MINOR} librocksdbjni-linux64.so ${exebin}
+          gzip  ${exebin}
+          mv ${exebin}.gz java/target/dcompact_worker.gz
+
+      - name: Prerelease and Generate Checksums
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          set -xe
+          ROCKSDB_VERSION=`build_tools/version.sh full`
+          # ex: topling-8.10.2-frocksdb-1.0, part <frocksdb> will be ignored
+          TOPLING_VERSION=`echo ${GITHUB_REF} | sed 's:^refs/tags/topling-'${ROCKSDB_VERSION}'[-_a-z]*\([.0-9]\):\1:'`
+          ROCKSDB_JAVA_VERSION=${ROCKSDB_VERSION}-topling-${TOPLING_VERSION}
+          cd java/target
+          db_artifactId=`sed -n 's/.*<artifactId>\(f\?rocksdbjni\)<\/artifactId>.*/\1/p' ../pom.xml.template`
+          TARGET_JAR=${db_artifactId}-${ROCKSDB_JAVA_VERSION}.jar
+          mv rocksdbjni-${ROCKSDB_VERSION}-linux64.jar ${TARGET_JAR}
+          shasum -a 1 ${TARGET_JAR} > ${TARGET_JAR}.sha1
+          md5sum      ${TARGET_JAR} > ${TARGET_JAR}.md5
+          gh release upload --clobber ${{github.ref_name}} ${TARGET_JAR}* index.html style.css dcompact_worker.gz