diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml new file mode 100644 index 0000000..fca453e --- /dev/null +++ b/.github/workflows/bitcoin-core-ci.yml @@ -0,0 +1,421 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +# Test libmultiprocess inside Bitcoin Core by replacing the subtree copy +# with the version from this PR, then building and running IPC-related +# unit & functional tests. + +name: Bitcoin Core CI + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BITCOIN_REPO: bitcoin/bitcoin + LLVM_VERSION: 22 + LIBCXX_DIR: /tmp/libcxx-build/ + ASAN_UBSAN_UNIT_TEST_RUNS: 15 + ASAN_UBSAN_FUNCTIONAL_TEST_RUNS: 300 + ASAN_UBSAN_NPROC_MULTIPLIER: 10 + MACOS_UNIT_TEST_RUNS: 50 + MACOS_FUNCTIONAL_TEST_RUNS: 200 + MACOS_NPROC_MULTIPLIER: 5 + TSAN_UNIT_TEST_RUNS: 8 + TSAN_FUNCTIONAL_TEST_RUNS: 300 + TSAN_NPROC_MULTIPLIER: 10 + +jobs: + bitcoin-core: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: 'ASan + UBSan' + runner: ubuntu-24.04 + apt-llvm: true + packages: >- + ccache + clang-22 + llvm-22 + libclang-rt-22-dev + libevent-dev + libboost-dev + libsqlite3-dev + libcapnp-dev + capnproto + ninja-build + pkgconf + python3-pip + pip-packages: --break-system-packages pycapnp + cmake-args: >- + -DSANITIZERS=address,float-divide-by-zero,integer,undefined + -DCMAKE_C_COMPILER=clang + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' + -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' + + - name: 'macOS' + runner: macos-15 + brew-packages: ccache capnp boost libevent sqlite pkgconf ninja + pip-packages: --break-system-packages pycapnp + cmake-args: >- + -DREDUCE_EXPORTS=ON + + env: + CCACHE_MAXSIZE: 400M + CCACHE_DIR: ${{ github.workspace }}/.ccache + + steps: + - name: Checkout Bitcoin Core + uses: actions/checkout@v4 + with: + repository: ${{ env.BITCOIN_REPO }} + fetch-depth: 1 + + - name: Checkout libmultiprocess + uses: actions/checkout@v4 + with: + path: _libmultiprocess + + - name: Replace libmultiprocess subtree + run: | + rm -rf src/ipc/libmultiprocess + mv _libmultiprocess src/ipc/libmultiprocess + + - name: Add LLVM apt repository + if: matrix.apt-llvm + run: | + curl -s "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" > /dev/null + source /etc/os-release + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VERSION} main" | sudo tee "/etc/apt/sources.list.d/llvm.list" + sudo apt-get update + + - name: Install APT packages + if: matrix.packages + run: | + sudo apt-get install --no-install-recommends -y ${{ matrix.packages }} + sudo update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" 100 + + - name: Install Homebrew packages + if: matrix.brew-packages + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + run: | + brew install --quiet ${{ matrix.brew-packages }} + + - name: Install pip packages + if: matrix.pip-packages + run: pip3 install ${{ matrix.pip-packages }} + + - name: Determine parallelism + run: | + if command -v nproc >/dev/null 2>&1; then + available_nproc="$(nproc)" + else + available_nproc="$(sysctl -n hw.logicalcpu)" + fi + echo "BUILD_PARALLEL=${available_nproc}" >> "$GITHUB_ENV" + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + nproc_multiplier="${ASAN_UBSAN_NPROC_MULTIPLIER}" + else + nproc_multiplier="${MACOS_NPROC_MULTIPLIER}" + fi + echo "PARALLEL=$((available_nproc * nproc_multiplier))" >> "$GITHUB_ENV" + + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ matrix.name }}-${{ github.ref }}- + ccache-${{ matrix.name }}- + + - name: Reset ccache stats + if: matrix.packages || matrix.brew-packages + run: | + which ccache + ccache --version + ccache --zero-stats + + - name: CMake configure + run: | + cmake -S . -B build \ + --preset=dev-mode \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_GUI=OFF \ + -DBUILD_GUI_TESTS=OFF \ + -DWITH_ZMQ=OFF \ + -DWITH_USDT=OFF \ + -DBUILD_BENCH=OFF \ + -DBUILD_FUZZ_BINARY=OFF \ + -DWITH_QRENCODE=OFF \ + -G Ninja \ + ${{ matrix.cmake-args }} + + - name: Build + run: cmake --build build --parallel "${BUILD_PARALLEL}" + + - name: Show ccache stats + if: matrix.packages || matrix.brew-packages + run: ccache --show-stats + + - name: Run IPC unit tests + env: + ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan + UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 + run: | + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${ASAN_UBSAN_UNIT_TEST_RUNS}" + else + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${MACOS_UNIT_TEST_RUNS}" + fi + + - name: Run IPC functional tests + env: + ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan + UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 + run: | + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${ASAN_UBSAN_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "4" + else + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${MACOS_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "4" + fi + + - name: Save ccache + uses: actions/cache/save@v4 + if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} + + bitcoin-core-tsan: + name: TSan + runs-on: ubuntu-24.04 + timeout-minutes: 180 + + env: + CCACHE_MAXSIZE: 400M + CCACHE_DIR: ${{ github.workspace }}/.ccache + LIBCXX_FLAGS: >- + -fsanitize=thread + -nostdinc++ + -nostdlib++ + -isystem /tmp/libcxx-build/include/c++/v1 + -L/tmp/libcxx-build/lib + -Wl,-rpath,/tmp/libcxx-build/lib + -lc++ + -lc++abi + -lpthread + -Wno-unused-command-line-argument + TSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1 + + steps: + - name: Checkout Bitcoin Core + uses: actions/checkout@v4 + with: + repository: ${{ env.BITCOIN_REPO }} + fetch-depth: 1 + + - name: Add LLVM apt repository + run: | + curl -s "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" > /dev/null + source /etc/os-release + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VERSION} main" | sudo tee "/etc/apt/sources.list.d/llvm.list" + sudo apt-get update + + - name: Install packages + run: | + sudo apt-get install --no-install-recommends -y \ + ccache \ + "clang-${LLVM_VERSION}" \ + "llvm-${LLVM_VERSION}" \ + "llvm-${LLVM_VERSION}-dev" \ + "libclang-${LLVM_VERSION}-dev" \ + "libclang-rt-${LLVM_VERSION}-dev" \ + ninja-build \ + pkgconf \ + python3-pip \ + bison + sudo update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" 100 + sudo update-alternatives --set clang "/usr/bin/clang-${LLVM_VERSION}" + sudo update-alternatives --set clang++ "/usr/bin/clang++-${LLVM_VERSION}" + sudo update-alternatives --set llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" + pip3 install --break-system-packages pycapnp + + - name: Determine parallelism + run: | + if command -v nproc >/dev/null 2>&1; then + available_nproc="$(nproc)" + else + available_nproc="$(sysctl -n hw.logicalcpu)" + fi + echo "BUILD_PARALLEL=${available_nproc}" >> "$GITHUB_ENV" + echo "PARALLEL=$((available_nproc * TSAN_NPROC_MULTIPLIER))" >> "$GITHUB_ENV" + + - name: Restore instrumented libc++ cache + id: libcxx-cache + uses: actions/cache@v4 + with: + path: ${{ env.LIBCXX_DIR }} + key: libcxx-Thread-llvmorg-${{ env.LLVM_VERSION }}.1.0 + + - name: Build instrumented libc++ + if: steps.libcxx-cache.outputs.cache-hit != 'true' + run: | + export PATH="/usr/lib/llvm-${LLVM_VERSION}/bin:$PATH" + ls -l /usr/bin/clang /usr/bin/clang++ /usr/bin/llvm-symbolizer + which clang clang++ llvm-symbolizer + clang --version + clang++ --version + "/usr/bin/clang-${LLVM_VERSION}" --version + "/usr/bin/clang++-${LLVM_VERSION}" --version + git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-${LLVM_VERSION}.1.0" /tmp/llvm-project + cmake -G Ninja -B "$LIBCXX_DIR" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=Thread \ + -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" \ + -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ + -S /tmp/llvm-project/runtimes + grep -E 'CMAKE_(C|CXX)_COMPILER' "$LIBCXX_DIR/CMakeCache.txt" + ninja -C "$LIBCXX_DIR" -j "${BUILD_PARALLEL}" -v + rm -rf /tmp/llvm-project + + - name: Determine host + id: host + run: echo "host=$(./depends/config.guess)" >> "$GITHUB_OUTPUT" + + - name: Restore depends cache + id: depends-cache + uses: actions/cache/restore@v4 + with: + path: | + depends/built + depends/${{ steps.host.outputs.host }} + key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} + + - name: Build depends (stage 1, without IPC) + if: steps.depends-cache.outputs.cache-hit != 'true' + run: | + make -C depends -j "${BUILD_PARALLEL}" \ + CC=clang \ + CXX=clang++ \ + CXXFLAGS="${LIBCXX_FLAGS}" \ + NO_QT=1 \ + NO_ZMQ=1 \ + NO_USDT=1 \ + NO_QR=1 \ + NO_IPC=1 + + - name: Save depends cache + uses: actions/cache/save@v4 + if: steps.depends-cache.outputs.cache-hit != 'true' + with: + path: | + depends/built + depends/${{ steps.host.outputs.host }} + key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} + + - name: Checkout libmultiprocess + uses: actions/checkout@v4 + with: + path: _libmultiprocess + + - name: Replace libmultiprocess subtree + run: | + rm -rf src/ipc/libmultiprocess + mv _libmultiprocess src/ipc/libmultiprocess + + - name: Build depends (stage 2, IPC packages including libmultiprocess) + run: | + make -C depends -j "${BUILD_PARALLEL}" \ + CC=clang \ + CXX=clang++ \ + CXXFLAGS="${LIBCXX_FLAGS}" \ + NO_QT=1 \ + NO_ZMQ=1 \ + NO_USDT=1 \ + NO_QR=1 + + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-TSan-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-TSan-${{ github.ref }}- + ccache-TSan- + + - name: Reset ccache stats + run: | + which ccache + ccache --version + ccache --zero-stats + + - name: CMake configure + run: | + cmake -S . -B build \ + --preset=dev-mode \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_GUI=OFF \ + -DBUILD_GUI_TESTS=OFF \ + -DWITH_ZMQ=OFF \ + -DWITH_USDT=OFF \ + -DBUILD_BENCH=OFF \ + -DBUILD_FUZZ_BINARY=OFF \ + -DWITH_QRENCODE=OFF \ + -DSANITIZERS=thread \ + -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES' \ + -DCMAKE_TOOLCHAIN_FILE=depends/${{ steps.host.outputs.host }}/toolchain.cmake \ + -G Ninja + + - name: Build + run: cmake --build build --parallel "${BUILD_PARALLEL}" + + - name: Show ccache stats + run: ccache --show-stats + + - name: Run IPC unit tests + run: | + LD_LIBRARY_PATH="depends/${{ steps.host.outputs.host }}/lib" \ + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${TSAN_UNIT_TEST_RUNS}" + + - name: Run IPC functional tests + env: + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 + run: | + LD_LIBRARY_PATH="depends/${{ steps.host.outputs.host }}/lib" \ + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${TSAN_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "10" + + - name: Save ccache + uses: actions/cache/save@v4 + if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-TSan-${{ github.ref }}-${{ github.sha }} diff --git a/ci/scripts/run_bitcoin_core_functional_tests.sh b/ci/scripts/run_bitcoin_core_functional_tests.sh new file mode 100755 index 0000000..e05d578 --- /dev/null +++ b/ci/scripts/run_bitcoin_core_functional_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +RUNS=$1 +PARALLEL=$2 +TIMEOUT_FACTOR=$3 + +test_scripts=$(python3 -c "import sys; import os; sys.path.append(os.path.abspath('build/test/functional')); from test_runner import ALL_SCRIPTS; print(' '.join(s for s in ALL_SCRIPTS if s.startswith('interface_ipc')))") +test_args=() +for _ in $(seq 1 "${RUNS}"); do + for script in $test_scripts; do + test_args+=("$script") + done +done +build/test/functional/test_runner.py "${test_args[@]}" --jobs "${PARALLEL}" --timeout-factor="${TIMEOUT_FACTOR}" --failfast --combinedlogslen=99999999 diff --git a/ci/scripts/run_bitcoin_core_unit_tests.sh b/ci/scripts/run_bitcoin_core_unit_tests.sh new file mode 100755 index 0000000..fb50ede --- /dev/null +++ b/ci/scripts/run_bitcoin_core_unit_tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex + +RUNS=$1 + +for _ in $(seq 1 "${RUNS}"); do + build/bin/test_bitcoin --run_test=ipc_tests,miner_tests --catch_system_error=no --log_level=nothing --report_level=no +done