From 4b95861fa78bf387fb82083d21e5b63dbde98155 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 20 May 2026 14:21:22 +0000 Subject: [PATCH 1/3] ref: build cuDF harness from source in CI Fetch and build cuDF 26.04.00 through CMake instead of relying on a conda-provided libcudf. Update CI to use the CUDA 13.1 Ubuntu image, install CMake from apt, build with Ninja, and verify the release bundle runs with an $ORIGIN/lib RPATH. --- .github/workflows/release.yml | 58 ++++++++++++++++++++++------------- CLAUDE.md | 10 ++++-- CMakeLists.txt | 39 +++++++++++++++++------ scripts/bundle.sh | 13 ++------ 4 files changed, 74 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29075b7..7870200 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,7 @@ name: Build and Release on: + pull_request: push: tags: - 'v*' @@ -8,44 +9,57 @@ on: jobs: build: + name: Build and Release runs-on: ubuntu-latest container: - image: rapidsai/ci-conda:latest + image: nvidia/cuda:13.1.0-devel-ubuntu24.04 steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup conda environment + - name: Install build tools run: | - conda create -n build-env -y -c rapidsai -c conda-forge -c nvidia \ - libcudf=26.04.00 \ + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl + curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc -o /usr/share/keyrings/kitware-archive-keyring.asc + echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.asc] https://apt.kitware.com/ubuntu/ noble main' > /etc/apt/sources.list.d/kitware.list + apt-get update + apt-get install -y --no-install-recommends \ + build-essential \ cmake \ - ninja \ - patchelf \ - gcc_linux-64 \ - gxx_linux-64 + git \ + ninja-build \ + tar \ + zlib1g-dev + cmake --version + /usr/local/cuda/bin/nvcc --version + + - name: Checkout + uses: actions/checkout@v4 - name: Build - shell: bash -l {0} run: | - source /opt/conda/etc/profile.d/conda.sh - conda activate build-env - mkdir build && cd build - cmake -G Ninja \ + cmake -S . -B build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST_LIB=OFF \ - .. - ninja + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DBUILD_TEST_LIB=OFF + cmake --build build - name: Create bundle - shell: bash -l {0} run: | - source /opt/conda/etc/profile.d/conda.sh - conda activate build-env chmod +x scripts/bundle.sh ./scripts/bundle.sh + - name: Verify bundle + run: | + rm -rf /tmp/cudf-test-harness-bundle + mkdir -p /tmp/cudf-test-harness-bundle + tar -xzf build/bundle/cudf-test-harness-*.tar.gz -C /tmp/cudf-test-harness-bundle + BINARY="$(echo /tmp/cudf-test-harness-bundle/*/cudf-test-harness)" + readelf -d "$BINARY" | grep -F '$ORIGIN/lib' + OUTPUT="$("$BINARY" 2>&1 || true)" + echo "$OUTPUT" | grep 'Usage:' + - name: Upload artifact uses: actions/upload-artifact@v4 with: diff --git a/CLAUDE.md b/CLAUDE.md index c084d0f..19dd848 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,13 +28,17 @@ and a non-zero exit code should be returned. ## Building -Expect that this is being build in a CMake project, inside of a conda environment where the following has been run: +Build with CMake: ``` -conda install -c rapidsai -c conda-forge -c nvidia rapidsai::libcudf +cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DBUILD_TEST_LIB=OFF +cmake --build build ``` -The binary should build in debug mode with full debug symbols by default. +The CMake configure step fetches and builds cuDF `26.04.00` from source, including the dependencies managed by cuDF's RAPIDS CMake files. This source-build path requires CMake 3.30.4 or newer and CUDA Toolkit 13.0 or newer. The binary builds in debug mode with full debug symbols by default when `CMAKE_BUILD_TYPE` is not set. ## Links diff --git a/CMakeLists.txt b/CMakeLists.txt index 42b9e91..e8f84c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ -cmake_minimum_required(VERSION 3.26) +cmake_minimum_required(VERSION 3.30.4) -project(cudf-test-harness LANGUAGES CXX CUDA) +project(cudf-test-harness LANGUAGES C CXX CUDA) # Default to Debug build with full debug symbols if(NOT CMAKE_BUILD_TYPE) @@ -9,13 +9,36 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_SCAN_FOR_MODULES OFF) # Debug flags set(CMAKE_CXX_FLAGS_DEBUG "-g3 -O0") set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0") -# Find cuDF -find_package(cudf REQUIRED) +include(FetchContent) +find_package(CUDAToolkit REQUIRED) + +if(CUDAToolkit_VERSION VERSION_LESS 13.0) + message(FATAL_ERROR "cuDF 26.04.00 source builds require CUDA Toolkit 13.0 or newer") +endif() + +foreach(option + BUILD_TESTS + BUILD_BENCHMARKS + CUDF_BUILD_TESTUTIL + CUDF_BUILD_STREAMS_TEST_UTIL + CUDF_KVIKIO_REMOTE_IO +) + set(${option} OFF CACHE BOOL "Disabled for cudf-test-harness cuDF source build" FORCE) +endforeach() + +FetchContent_Declare( + cudf + GIT_REPOSITORY https://github.com/rapidsai/cudf.git + GIT_TAG f9c3cf195768647ac39d98674a614ca414cd1baa + SOURCE_SUBDIR cpp +) +FetchContent_MakeAvailable(cudf) # Create the executable add_executable(cudf-test-harness @@ -26,25 +49,21 @@ target_include_directories(cudf-test-harness PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ) -# Use the conda environment's libstdc++ to match cuDF -# Must be linked before other libraries to override system libstdc++ target_link_libraries(cudf-test-harness PRIVATE - $ENV{CONDA_PREFIX}/lib/libstdc++.so cudf::cudf ${CMAKE_DL_LIBS} ) # Set RPATH for bundled distribution (looks for libs relative to binary) -# During build, also include conda lib path set_target_properties(cudf-test-harness PROPERTIES - BUILD_RPATH "$ENV{CONDA_PREFIX}/lib" INSTALL_RPATH "$ORIGIN/lib" BUILD_WITH_INSTALL_RPATH OFF ) # Install target install(TARGETS cudf-test-harness - RUNTIME DESTINATION bin + RUNTIME DESTINATION . + COMPONENT harness ) # Optional: Build test library diff --git a/scripts/bundle.sh b/scripts/bundle.sh index 6c3f1a6..7cc359e 100755 --- a/scripts/bundle.sh +++ b/scripts/bundle.sh @@ -26,8 +26,8 @@ mkdir -p "$BUNDLE_PATH/lib" echo "Collecting dependencies for $BINARY..." -# Copy the binary -cp "$BINARY" "$BUNDLE_PATH/" +# Install the binary so CMake applies the bundle RPATH. +cmake --install "$BUILD_DIR" --prefix "$BUNDLE_PATH" --component harness # Get all shared library dependencies (excluding system libraries we expect to exist) # We keep: libcu*, libnv*, librmm*, libcudf*, libarrow*, libstdc++, etc. @@ -88,15 +88,6 @@ exec "${SCRIPT_DIR}/cudf-test-harness" "$@" WRAPPER_EOF chmod +x "$BUNDLE_PATH/cudf-test-harness.sh" -# Also patch the binary's rpath to look in ./lib (for direct execution) -if command -v patchelf &> /dev/null; then - echo "Patching rpath..." - patchelf --set-rpath '$ORIGIN/lib' "$BUNDLE_PATH/cudf-test-harness" -else - echo "Warning: patchelf not found. Install it for direct binary execution." - echo " Without it, use the wrapper script: ./cudf-test-harness.sh" -fi - # Create tarball echo "" echo "Creating tarball..." From d8d951ee73de420f41a91354229e7823c09e38ff Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 20 May 2026 14:43:45 +0000 Subject: [PATCH 2/3] sccache --- .github/workflows/release.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7870200..2c22256 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,11 +10,15 @@ on: jobs: build: name: Build and Release - runs-on: ubuntu-latest + runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cudf-test-harness container: image: nvidia/cuda:13.1.0-devel-ubuntu24.04 steps: + - uses: runs-on/action@v2 + with: + sccache: s3 + - name: Install build tools run: | apt-get update @@ -42,6 +46,9 @@ jobs: cmake -S . -B build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CUDA_COMPILER_LAUNCHER=sccache \ -DBUILD_TEST_LIB=OFF cmake --build build From edf46fdd4570e24d4d46d6328848d561da0b3cd8 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 20 May 2026 14:47:29 +0000 Subject: [PATCH 3/3] name --- .github/workflows/{release.yml => ci.yml} | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) rename .github/workflows/{release.yml => ci.yml} (85%) diff --git a/.github/workflows/release.yml b/.github/workflows/ci.yml similarity index 85% rename from .github/workflows/release.yml rename to .github/workflows/ci.yml index 2c22256..89accdf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: push: tags: - - 'v*' + - '*' workflow_dispatch: jobs: @@ -19,6 +19,17 @@ jobs: with: sccache: s3 + - name: Configure sccache timeout + run: | + mkdir -p ~/.config/sccache + echo 'server_startup_timeout_ms = 60000' > ~/.config/sccache/config + + - name: Install sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + + - name: Pre-start sccache server + run: sccache --start-server + - name: Install build tools run: | apt-get update @@ -68,6 +79,7 @@ jobs: echo "$OUTPUT" | grep 'Usage:' - name: Upload artifact + if: startsWith(github.ref, 'refs/tags/') uses: actions/upload-artifact@v4 with: name: cudf-test-harness-linux-x86_64