From 88cacd4239fc3fd147b22c2ad3091c5123c3285f Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 11 Mar 2026 13:39:38 +0100 Subject: [PATCH 1/9] test: worker thread destroyed before it is initialized Add test for race condition in makeThread that can currently trigger segfaults as reported: https://github.com/bitcoin/bitcoin/issues/34711 https://github.com/bitcoin/bitcoin/issues/34756 The test currently crashes and will be fixed in the next commit. Co-authored-by: Ryan Ofsky git-bisect-skip: yes --- include/mp/proxy-io.h | 8 ++++++++ src/mp/proxy.cpp | 5 ++++- test/mp/test/test.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index c298257..9399f54 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -340,6 +340,14 @@ class EventLoop //! External context pointer. void* m_context; + + //! Hook called when ProxyServer::makeThread() is called. + std::function testing_hook_makethread; + + //! Hook called on the worker thread inside makeThread(), after the thread + //! context is set up and thread_context promise is fulfilled, but before it + //! starts waiting for requests. + std::function testing_hook_makethread_created; }; //! Single element task queue used to handle recursive capnp calls. (If the diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index da22ae6..74063ea 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -411,12 +411,15 @@ ProxyServer::ProxyServer(Connection& connection) : m_connection(conne kj::Promise ProxyServer::makeThread(MakeThreadContext context) { + if (m_connection.m_loop->testing_hook_makethread) m_connection.m_loop->testing_hook_makethread(); const std::string from = context.getParams().getName(); std::promise thread_context; std::thread thread([&thread_context, from, this]() { - g_thread_context.thread_name = ThreadName(m_connection.m_loop->m_exe_name) + " (from " + from + ")"; + EventLoop& loop{*m_connection.m_loop}; + g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; g_thread_context.waiter = std::make_unique(); thread_context.set_value(&g_thread_context); + if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); Lock lock(g_thread_context.waiter->m_mutex); // Wait for shutdown signal from ProxyServer destructor (signal // is just waiter getting set to null.) diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index bf41663..1fd5908 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -63,6 +64,7 @@ class TestSetup { public: std::function server_disconnect; + std::function server_disconnect_later; std::function client_disconnect; std::promise>> client_promise; std::unique_ptr> client; @@ -88,6 +90,10 @@ class TestSetup return capnp::Capability::Client(kj::mv(server_proxy)); }); server_disconnect = [&] { loop.sync([&] { server_connection.reset(); }); }; + server_disconnect_later = [&] { + assert(std::this_thread::get_id() == loop.m_thread_id); + loop.m_task_set->add(kj::evalLater([&] { server_connection.reset(); })); + }; // Set handler to destroy the server when the client disconnects. This // is ignored if server_disconnect() is called instead. server_connection->onDisconnect([&] { server_connection.reset(); }); @@ -325,6 +331,40 @@ KJ_TEST("Calling IPC method, disconnecting and blocking during the call") signal.set_value(); } +KJ_TEST("Worker thread destroyed before it is initialized") +{ + // Regression test for bitcoin/bitcoin#34711, bitcoin/bitcoin#34756 + // where worker thread is destroyed before it starts. + // + // The test works by using the `makethread` hook to start a disconnect as + // soon as ProxyServer::makeThread is called, and using the + // `makethread_created` hook to sleep 100ms after the thread is created but + // before it starts waiting, so without the bugfix, + // ProxyServer::~ProxyServer would run and destroy the waiter, + // causing a SIGSEGV in the worker thread after the sleep. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_fn = [] {}; + + EventLoop& loop = *setup.server->m_context.connection->m_loop; + loop.testing_hook_makethread = [&] { + setup.server_disconnect_later(); + }; + loop.testing_hook_makethread_created = [&] { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + }; + + bool disconnected{false}; + try { + foo->callFnAsync(); + } catch (const std::runtime_error& e) { + KJ_EXPECT(std::string_view{e.what()} == "IPC client method call interrupted by disconnect."); + disconnected = true; + } + KJ_EXPECT(disconnected); +} + KJ_TEST("Make simultaneous IPC calls on single remote thread") { TestSetup setup; From f09731e242f30d92873fb332419f162ece253328 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 2 Mar 2026 14:20:44 -0500 Subject: [PATCH 2/9] race fix: worker thread destroyed before it is initialized This fixes a race condition in makeThread that can currently trigger segfaults as reported: https://github.com/bitcoin/bitcoin/issues/34711 https://github.com/bitcoin/bitcoin/issues/34756 The bug can be reproduced by running the unit test added in the previous commit or by calling makeThread and immediately disconnecting or destroying the returned thread. The bug is not new and has existed since makeThread was implemented, but it was found due to a new functional test in bitcoin core and with antithesis testing (see details in linked issues). The fix was originally posted in https://github.com/bitcoin/bitcoin/issues/34711#issuecomment-3986380070 --- src/mp/proxy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mp/proxy.cpp b/src/mp/proxy.cpp index 74063ea..f36e19f 100644 --- a/src/mp/proxy.cpp +++ b/src/mp/proxy.cpp @@ -418,9 +418,9 @@ kj::Promise ProxyServer::makeThread(MakeThreadContext context) EventLoop& loop{*m_connection.m_loop}; g_thread_context.thread_name = ThreadName(loop.m_exe_name) + " (from " + from + ")"; g_thread_context.waiter = std::make_unique(); + Lock lock(g_thread_context.waiter->m_mutex); thread_context.set_value(&g_thread_context); if (loop.testing_hook_makethread_created) loop.testing_hook_makethread_created(); - Lock lock(g_thread_context.waiter->m_mutex); // Wait for shutdown signal from ProxyServer destructor (signal // is just waiter getting set to null.) g_thread_context.waiter->wait(lock, [] { return !g_thread_context.waiter; }); From 75c5425173fb517755252ff20b4af8d1342972db Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Tue, 10 Mar 2026 13:34:03 -0400 Subject: [PATCH 3/9] test: getParams() called after request cancel Add test for disconnect race condition in the mp.Context PassField() overload that can currently trigger segfaults as reported in https://github.com/bitcoin/bitcoin/issues/34777. The test currently crashes and will be fixed in the next commit. Co-authored-by: Ryan Ofsky git-bisect-skip: yes --- include/mp/proxy-io.h | 5 +++++ include/mp/type-context.h | 2 ++ test/mp/test/test.cpp | 30 ++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index 9399f54..b1e474e 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -348,6 +348,11 @@ class EventLoop //! context is set up and thread_context promise is fulfilled, but before it //! starts waiting for requests. std::function testing_hook_makethread_created; + + //! Hook called on the worker thread when it starts to execute an async + //! request. Used by tests to control timing or inject behavior at this + //! point in execution. + std::function testing_hook_async_request_start; }; //! Single element task queue used to handle recursive capnp calls. (If the diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 72c3963..86a80d3 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -74,6 +74,8 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& auto self = server.thisCap(); auto invoke = [self = kj::mv(self), call_context = kj::mv(server_context.call_context), &server, req, fn, args...](CancelMonitor& cancel_monitor) mutable { MP_LOG(*server.m_context.loop, Log::Debug) << "IPC server executing request #" << req; + EventLoop& loop = *server.m_context.loop; + if (loop.testing_hook_async_request_start) loop.testing_hook_async_request_start(); const auto& params = call_context.getParams(); Context::Reader context_arg = Accessor::get(params); ServerContext server_context{server, call_context, req}; diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 1fd5908..65e48d7 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -365,6 +365,36 @@ KJ_TEST("Worker thread destroyed before it is initialized") KJ_EXPECT(disconnected); } +KJ_TEST("Calling async IPC method, with server disconnect racing the call") +{ + // Regression test for bitcoin/bitcoin#34777 heap-use-after-free where + // an async request is canceled before it starts to execute. + // + // Use testing_hook_async_request_start to trigger a disconnect from the + // worker thread as soon as it begins to execute an async request. Without + // the bugfix, the worker thread would trigger a SIGSEGV after this by + // calling call_context.getParams(). + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_fn = [] {}; + + EventLoop& loop = *setup.server->m_context.connection->m_loop; + loop.testing_hook_async_request_start = [&] { + setup.server_disconnect(); + // Sleep is neccessary to let the event loop fully clean up after the + // disconnect and trigger the SIGSEGV. + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + }; + + try { + foo->callFnAsync(); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + KJ_EXPECT(std::string_view{e.what()} == "IPC client method call interrupted by disconnect."); + } +} + KJ_TEST("Make simultaneous IPC calls on single remote thread") { TestSetup setup; From e69b6bf3f4ef6ba45606cf85fdb44cc4607c021e Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Mon, 9 Mar 2026 10:43:37 -0400 Subject: [PATCH 4/9] race fix: getParams() called after request cancel This fixes a race condition in the mp.Context PassField() overload which is used to execute async requests, that can currently trigger segfaults as reported in https://github.com/bitcoin/bitcoin/issues/34777 when it calls call_context.getParams() after a disconnect. The bug can be reproduced by running the unit test added in the previous commit and was also seen in antithesis (see details in linked issue), but should be unlikely to happen normally because PassField checks for cancellation and returns early before actually using the getParams() result. This bug was introduced commit in 0174450ca2e95a4bd1f22e4fd38d83b1d432ac1f which started to cancel requests on disconnects. Before that commit, requests would continue to execute after a disconnect and it was ok to call getParams(). This fix was originally posted in https://github.com/bitcoin/bitcoin/issues/34777#issuecomment-4024285314 --- include/mp/type-context.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 86a80d3..18057e5 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -61,8 +61,6 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& std::is_same::value, kj::Promise>::type { - const auto& params = server_context.call_context.getParams(); - Context::Reader context_arg = Accessor::get(params); auto& server = server_context.proxy_server; int req = server_context.req; // Keep a reference to the ProxyServer instance by assigning it to the self @@ -76,8 +74,6 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& MP_LOG(*server.m_context.loop, Log::Debug) << "IPC server executing request #" << req; EventLoop& loop = *server.m_context.loop; if (loop.testing_hook_async_request_start) loop.testing_hook_async_request_start(); - const auto& params = call_context.getParams(); - Context::Reader context_arg = Accessor::get(params); ServerContext server_context{server, call_context, req}; { // Before invoking the function, store a reference to the @@ -129,6 +125,8 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& server_context.request_canceled = true; }; // Update requests_threads map if not canceled. + const auto& params = call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); std::tie(request_thread, inserted) = SetThread( GuardedRef{thread_context.waiter->m_mutex, request_threads}, server.m_context.connection, [&] { return context_arg.getCallbackThread(); }); @@ -191,6 +189,8 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // Lookup Thread object specified by the client. The specified thread should // be a local Thread::Server object, but it needs to be looked up // asynchronously with getLocalServer(). + const auto& params = server_context.call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); auto thread_client = context_arg.getThread(); auto result = server.m_context.connection->m_threads.getLocalServer(thread_client) .then([&server, invoke = kj::mv(invoke), req](const kj::Maybe& perhaps) mutable { From 846a43aafb439e31aa4954e4be3e0920c2466877 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 11 Mar 2026 11:33:14 +0100 Subject: [PATCH 5/9] test: m_on_cancel called after request finishes Add test disconnect for race condition in the mp.Context PassField() overload reported in https://github.com/bitcoin/bitcoin/issues/34782. The test crashes currently with AddressSanitizer, but will be fixed in the next commit. It's also possible to reproduce the bug without AddressSanitizer by adding an assert: ```diff --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -101,2 +101,3 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& server_context.cancel_lock = &cancel_lock; + KJ_DEFER(server_context.cancel_lock = nullptr); server.m_context.loop->sync([&] { @@ -111,2 +112,3 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& MP_LOG(*server.m_context.loop, Log::Info) << "IPC server request #" << req << " canceled while executing."; + assert(server_context.cancel_lock); // Lock cancel_mutex here to block the event loop ``` Co-authored-by: Ryan Ofsky git-bisect-skip: yes --- include/mp/proxy-io.h | 3 +++ include/mp/type-context.h | 1 + test/mp/test/test.cpp | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/include/mp/proxy-io.h b/include/mp/proxy-io.h index b1e474e..3594708 100644 --- a/include/mp/proxy-io.h +++ b/include/mp/proxy-io.h @@ -353,6 +353,9 @@ class EventLoop //! request. Used by tests to control timing or inject behavior at this //! point in execution. std::function testing_hook_async_request_start; + + //! Hook called on the worker thread just before returning results. + std::function testing_hook_async_request_done; }; //! Single element task queue used to handle recursive capnp calls. (If the diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 18057e5..80ae71c 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -183,6 +183,7 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& } // End of scope: if KJ_DEFER was reached, it runs here } + if (loop.testing_hook_async_request_done) loop.testing_hook_async_request_done(); return call_context; }; diff --git a/test/mp/test/test.cpp b/test/mp/test/test.cpp index 65e48d7..4f71a55 100644 --- a/test/mp/test/test.cpp +++ b/test/mp/test/test.cpp @@ -395,6 +395,35 @@ KJ_TEST("Calling async IPC method, with server disconnect racing the call") } } +KJ_TEST("Calling async IPC method, with server disconnect after cleanup") +{ + // Regression test for bitcoin/bitcoin#34782 stack-use-after-return where + // an async request is canceled after it finishes executing but before the + // response is sent. + // + // Use testing_hook_async_request_done to trigger a disconnect from the + // worker thread after it execute an async requests but before it returns. + // Without the bugfix, the m_on_cancel callback would be called at this + // point accessing the cancel_mutex stack variable that had gone out of + // scope. + TestSetup setup; + ProxyClient* foo = setup.client.get(); + foo->initThreadMap(); + setup.server->m_impl->m_fn = [] {}; + + EventLoop& loop = *setup.server->m_context.connection->m_loop; + loop.testing_hook_async_request_done = [&] { + setup.server_disconnect(); + }; + + try { + foo->callFnAsync(); + KJ_EXPECT(false); + } catch (const std::runtime_error& e) { + KJ_EXPECT(std::string_view{e.what()} == "IPC client method call interrupted by disconnect."); + } +} + KJ_TEST("Make simultaneous IPC calls on single remote thread") { TestSetup setup; From 2fb97e8cca9feb1df70cf29b2a9895bea2c4c49c Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Tue, 10 Mar 2026 13:25:03 -0400 Subject: [PATCH 6/9] race fix: m_on_cancel called after request finishes This fixes a race condition in the mp.Context PassField() overload which is used to execute async requests, that can currently trigger segfaults as reported in https://github.com/bitcoin/bitcoin/issues/34782 when a cancellation happens after the request executes but before it returns. The bug can be reproduced by running the unit test added in the previous commit and was also seen in antithesis (see details in linked issue), but should be unlikely to happen normally because the cancellation would have to happen in a very short window for there to be a problem. This bug was introduced commit in 0174450ca2e95a4bd1f22e4fd38d83b1d432ac1f which started to cancel requests on disconnects. Before that commit a cancellation callback was not present. This fix was originally posted in https://github.com/bitcoin/bitcoin/issues/34782#issuecomment-4033169085 --- include/mp/type-context.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/mp/type-context.h b/include/mp/type-context.h index 80ae71c..9c7f21b 100644 --- a/include/mp/type-context.h +++ b/include/mp/type-context.h @@ -153,6 +153,15 @@ auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& // the disconnect handler trying to destroy the thread // client object. server.m_context.loop->sync([&] { + // Clear cancellation callback. At this point the + // method invocation finished and the result is + // either being returned, or discarded if a + // cancellation happened. So we do not need to be + // notified of cancellations after this point. Also + // we do not want to be notified because + // cancel_mutex and server_context could be out of + // scope when it happens. + cancel_monitor.m_on_cancel = nullptr; auto self_dispose{kj::mv(self)}; if (erase_thread) { // Look up the thread again without using existing From b78aa4a345fdb1793136e3af775bf56605c0533a Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 11 Mar 2026 17:10:03 +0100 Subject: [PATCH 7/9] ci: add Bitcoin Core IPC tests (ASan + macOS) --- .github/workflows/bitcoin-core-ci.yml | 202 ++++++++++++++++++ .../run_bitcoin_core_functional_tests.sh | 16 ++ ci/scripts/run_bitcoin_core_unit_tests.sh | 9 + 3 files changed, 227 insertions(+) create mode 100644 .github/workflows/bitcoin-core-ci.yml create mode 100755 ci/scripts/run_bitcoin_core_functional_tests.sh create mode 100755 ci/scripts/run_bitcoin_core_unit_tests.sh diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml new file mode 100644 index 0000000..e4b109b --- /dev/null +++ b/.github/workflows/bitcoin-core-ci.yml @@ -0,0 +1,202 @@ +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. + +# Test libmultiprocess inside Bitcoin Core by replacing the subtree copy +# with the version from this PR, then building and running IPC-related +# unit & functional tests. + +name: Bitcoin Core CI + +on: + push: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BITCOIN_REPO: bitcoin/bitcoin + LLVM_VERSION: 22 + ASAN_UBSAN_UNIT_TEST_RUNS: 1 + ASAN_UBSAN_FUNCTIONAL_TEST_RUNS: 1 + ASAN_UBSAN_NPROC_MULTIPLIER: 1 + MACOS_UNIT_TEST_RUNS: 1 + MACOS_FUNCTIONAL_TEST_RUNS: 1 + MACOS_NPROC_MULTIPLIER: 1 + +jobs: + bitcoin-core: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + include: + - name: 'ASan + UBSan' + runner: ubuntu-24.04 + apt-llvm: true + packages: >- + ccache + clang-22 + llvm-22 + libclang-rt-22-dev + libevent-dev + libboost-dev + libsqlite3-dev + libcapnp-dev + capnproto + ninja-build + pkgconf + python3-pip + pip-packages: --break-system-packages pycapnp + cmake-args: >- + -DSANITIZERS=address,float-divide-by-zero,integer,undefined + -DCMAKE_C_COMPILER=clang + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' + -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' + + - name: 'macOS' + runner: macos-15 + brew-packages: ccache capnp boost libevent sqlite pkgconf ninja + pip-packages: --break-system-packages pycapnp + cmake-args: >- + -DREDUCE_EXPORTS=ON + + env: + CCACHE_MAXSIZE: 400M + CCACHE_DIR: ${{ github.workspace }}/.ccache + + steps: + - name: Checkout Bitcoin Core + uses: actions/checkout@v4 + with: + repository: ${{ env.BITCOIN_REPO }} + fetch-depth: 1 + + - name: Checkout libmultiprocess + uses: actions/checkout@v4 + with: + path: _libmultiprocess + + - name: Replace libmultiprocess subtree + run: | + rm -rf src/ipc/libmultiprocess + mv _libmultiprocess src/ipc/libmultiprocess + + - name: Add LLVM apt repository + if: matrix.apt-llvm + run: | + curl -s "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" > /dev/null + source /etc/os-release + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VERSION} main" | sudo tee "/etc/apt/sources.list.d/llvm.list" + sudo apt-get update + + - name: Install APT packages + if: matrix.packages + run: | + sudo apt-get install --no-install-recommends -y ${{ matrix.packages }} + sudo update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" 100 + + - name: Install Homebrew packages + if: matrix.brew-packages + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + run: | + brew install --quiet ${{ matrix.brew-packages }} + + - name: Install pip packages + if: matrix.pip-packages + run: pip3 install ${{ matrix.pip-packages }} + + - name: Determine parallelism + run: | + if command -v nproc >/dev/null 2>&1; then + available_nproc="$(nproc)" + else + available_nproc="$(sysctl -n hw.logicalcpu)" + fi + echo "BUILD_PARALLEL=${available_nproc}" >> "$GITHUB_ENV" + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + nproc_multiplier="${ASAN_UBSAN_NPROC_MULTIPLIER}" + else + nproc_multiplier="${MACOS_NPROC_MULTIPLIER}" + fi + echo "PARALLEL=$((available_nproc * nproc_multiplier))" >> "$GITHUB_ENV" + + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-${{ matrix.name }}-${{ github.ref }}- + ccache-${{ matrix.name }}- + + - name: Reset ccache stats + if: matrix.packages || matrix.brew-packages + run: | + which ccache + ccache --version + ccache --zero-stats + + - name: CMake configure + run: | + cmake -S . -B build \ + --preset=dev-mode \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_GUI=OFF \ + -DBUILD_GUI_TESTS=OFF \ + -DWITH_ZMQ=OFF \ + -DWITH_USDT=OFF \ + -DBUILD_BENCH=OFF \ + -DBUILD_FUZZ_BINARY=OFF \ + -DWITH_QRENCODE=OFF \ + -G Ninja \ + ${{ matrix.cmake-args }} + + - name: Build + run: cmake --build build --parallel "${BUILD_PARALLEL}" + + - name: Show ccache stats + if: matrix.packages || matrix.brew-packages + run: ccache --show-stats + + - name: Run IPC unit tests + env: + ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan + UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 + run: | + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${ASAN_UBSAN_UNIT_TEST_RUNS}" + else + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${MACOS_UNIT_TEST_RUNS}" + fi + + - name: Run IPC functional tests + env: + ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/lsan + UBSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1 + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 + run: | + if [[ "${{ matrix.name }}" == 'ASan + UBSan' ]]; then + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${ASAN_UBSAN_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "4" + else + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${MACOS_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "4" + fi + + - name: Save ccache + uses: actions/cache/save@v4 + if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} diff --git a/ci/scripts/run_bitcoin_core_functional_tests.sh b/ci/scripts/run_bitcoin_core_functional_tests.sh new file mode 100755 index 0000000..e05d578 --- /dev/null +++ b/ci/scripts/run_bitcoin_core_functional_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +RUNS=$1 +PARALLEL=$2 +TIMEOUT_FACTOR=$3 + +test_scripts=$(python3 -c "import sys; import os; sys.path.append(os.path.abspath('build/test/functional')); from test_runner import ALL_SCRIPTS; print(' '.join(s for s in ALL_SCRIPTS if s.startswith('interface_ipc')))") +test_args=() +for _ in $(seq 1 "${RUNS}"); do + for script in $test_scripts; do + test_args+=("$script") + done +done +build/test/functional/test_runner.py "${test_args[@]}" --jobs "${PARALLEL}" --timeout-factor="${TIMEOUT_FACTOR}" --failfast --combinedlogslen=99999999 diff --git a/ci/scripts/run_bitcoin_core_unit_tests.sh b/ci/scripts/run_bitcoin_core_unit_tests.sh new file mode 100755 index 0000000..fb50ede --- /dev/null +++ b/ci/scripts/run_bitcoin_core_unit_tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex + +RUNS=$1 + +for _ in $(seq 1 "${RUNS}"); do + build/bin/test_bitcoin --run_test=ipc_tests,miner_tests --catch_system_error=no --log_level=nothing --report_level=no +done From db756c844017423b06c8774f30ae5bf35bbb6d67 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 11 Mar 2026 17:16:56 +0100 Subject: [PATCH 8/9] ci: add TSan job with instrumented libc++ --- .github/workflows/bitcoin-core-ci.yml | 219 ++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index e4b109b..75fe947 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -19,12 +19,16 @@ concurrency: env: BITCOIN_REPO: bitcoin/bitcoin LLVM_VERSION: 22 + LIBCXX_DIR: /tmp/libcxx-build/ ASAN_UBSAN_UNIT_TEST_RUNS: 1 ASAN_UBSAN_FUNCTIONAL_TEST_RUNS: 1 ASAN_UBSAN_NPROC_MULTIPLIER: 1 MACOS_UNIT_TEST_RUNS: 1 MACOS_FUNCTIONAL_TEST_RUNS: 1 MACOS_NPROC_MULTIPLIER: 1 + TSAN_UNIT_TEST_RUNS: 1 + TSAN_FUNCTIONAL_TEST_RUNS: 1 + TSAN_NPROC_MULTIPLIER: 1 jobs: bitcoin-core: @@ -200,3 +204,218 @@ jobs: with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ matrix.name }}-${{ github.ref }}-${{ github.sha }} + + bitcoin-core-tsan: + name: TSan + runs-on: ubuntu-24.04 + timeout-minutes: 180 + + env: + CCACHE_MAXSIZE: 400M + CCACHE_DIR: ${{ github.workspace }}/.ccache + LIBCXX_FLAGS: >- + -fsanitize=thread + -nostdinc++ + -nostdlib++ + -isystem /tmp/libcxx-build/include/c++/v1 + -L/tmp/libcxx-build/lib + -Wl,-rpath,/tmp/libcxx-build/lib + -lc++ + -lc++abi + -lpthread + -Wno-unused-command-line-argument + TSAN_OPTIONS: suppressions=${{ github.workspace }}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1 + + steps: + - name: Checkout Bitcoin Core + uses: actions/checkout@v4 + with: + repository: ${{ env.BITCOIN_REPO }} + fetch-depth: 1 + + - name: Add LLVM apt repository + run: | + curl -s "https://apt.llvm.org/llvm-snapshot.gpg.key" | sudo tee "/etc/apt/trusted.gpg.d/apt.llvm.org.asc" > /dev/null + source /etc/os-release + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VERSION} main" | sudo tee "/etc/apt/sources.list.d/llvm.list" + sudo apt-get update + + - name: Install packages + run: | + sudo apt-get install --no-install-recommends -y \ + ccache \ + "clang-${LLVM_VERSION}" \ + "llvm-${LLVM_VERSION}" \ + "llvm-${LLVM_VERSION}-dev" \ + "libclang-${LLVM_VERSION}-dev" \ + "libclang-rt-${LLVM_VERSION}-dev" \ + ninja-build \ + pkgconf \ + python3-pip \ + bison + sudo update-alternatives --install /usr/bin/clang++ clang++ "/usr/bin/clang++-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/clang clang "/usr/bin/clang-${LLVM_VERSION}" 100 + sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" 100 + sudo update-alternatives --set clang "/usr/bin/clang-${LLVM_VERSION}" + sudo update-alternatives --set clang++ "/usr/bin/clang++-${LLVM_VERSION}" + sudo update-alternatives --set llvm-symbolizer "/usr/bin/llvm-symbolizer-${LLVM_VERSION}" + pip3 install --break-system-packages pycapnp + + - name: Determine parallelism + run: | + if command -v nproc >/dev/null 2>&1; then + available_nproc="$(nproc)" + else + available_nproc="$(sysctl -n hw.logicalcpu)" + fi + echo "BUILD_PARALLEL=${available_nproc}" >> "$GITHUB_ENV" + echo "PARALLEL=$((available_nproc * TSAN_NPROC_MULTIPLIER))" >> "$GITHUB_ENV" + + - name: Restore instrumented libc++ cache + id: libcxx-cache + uses: actions/cache@v4 + with: + path: ${{ env.LIBCXX_DIR }} + key: libcxx-Thread-llvmorg-${{ env.LLVM_VERSION }}.1.0 + + - name: Build instrumented libc++ + if: steps.libcxx-cache.outputs.cache-hit != 'true' + run: | + export PATH="/usr/lib/llvm-${LLVM_VERSION}/bin:$PATH" + ls -l /usr/bin/clang /usr/bin/clang++ /usr/bin/llvm-symbolizer + which clang clang++ llvm-symbolizer + clang --version + clang++ --version + "/usr/bin/clang-${LLVM_VERSION}" --version + "/usr/bin/clang++-${LLVM_VERSION}" --version + git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-${LLVM_VERSION}.1.0" /tmp/llvm-project + cmake -G Ninja -B "$LIBCXX_DIR" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_USE_SANITIZER=Thread \ + -DCMAKE_C_COMPILER="/usr/bin/clang-${LLVM_VERSION}" \ + -DCMAKE_CXX_COMPILER="/usr/bin/clang++-${LLVM_VERSION}" \ + -DLLVM_TARGETS_TO_BUILD=Native \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DLIBCXX_INCLUDE_TESTS=OFF \ + -DLIBCXXABI_INCLUDE_TESTS=OFF \ + -DLIBUNWIND_INCLUDE_TESTS=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ + -S /tmp/llvm-project/runtimes + grep -E 'CMAKE_(C|CXX)_COMPILER' "$LIBCXX_DIR/CMakeCache.txt" + ninja -C "$LIBCXX_DIR" -j "${BUILD_PARALLEL}" -v + rm -rf /tmp/llvm-project + + - name: Determine host + id: host + run: echo "host=$(./depends/config.guess)" >> "$GITHUB_OUTPUT" + + - name: Restore depends cache + id: depends-cache + uses: actions/cache/restore@v4 + with: + path: | + depends/built + depends/${{ steps.host.outputs.host }} + key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} + + - name: Build depends (stage 1, without IPC) + if: steps.depends-cache.outputs.cache-hit != 'true' + run: | + make -C depends -j "${BUILD_PARALLEL}" \ + CC=clang \ + CXX=clang++ \ + CXXFLAGS="${LIBCXX_FLAGS}" \ + NO_QT=1 \ + NO_ZMQ=1 \ + NO_USDT=1 \ + NO_QR=1 \ + NO_IPC=1 + + - name: Save depends cache + uses: actions/cache/save@v4 + if: steps.depends-cache.outputs.cache-hit != 'true' + with: + path: | + depends/built + depends/${{ steps.host.outputs.host }} + key: depends-tsan-${{ hashFiles('depends/packages/*.mk') }}-${{ env.LLVM_VERSION }} + + - name: Checkout libmultiprocess + uses: actions/checkout@v4 + with: + path: _libmultiprocess + + - name: Replace libmultiprocess subtree + run: | + rm -rf src/ipc/libmultiprocess + mv _libmultiprocess src/ipc/libmultiprocess + + - name: Build depends (stage 2, IPC packages including libmultiprocess) + run: | + make -C depends -j "${BUILD_PARALLEL}" \ + CC=clang \ + CXX=clang++ \ + CXXFLAGS="${LIBCXX_FLAGS}" \ + NO_QT=1 \ + NO_ZMQ=1 \ + NO_USDT=1 \ + NO_QR=1 + + - name: Restore ccache + id: ccache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-TSan-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ccache-TSan-${{ github.ref }}- + ccache-TSan- + + - name: Reset ccache stats + run: | + which ccache + ccache --version + ccache --zero-stats + + - name: CMake configure + run: | + cmake -S . -B build \ + --preset=dev-mode \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_GUI=OFF \ + -DBUILD_GUI_TESTS=OFF \ + -DWITH_ZMQ=OFF \ + -DWITH_USDT=OFF \ + -DBUILD_BENCH=OFF \ + -DBUILD_FUZZ_BINARY=OFF \ + -DWITH_QRENCODE=OFF \ + -DSANITIZERS=thread \ + -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES' \ + -DCMAKE_TOOLCHAIN_FILE=depends/${{ steps.host.outputs.host }}/toolchain.cmake \ + -G Ninja + + - name: Build + run: cmake --build build --parallel "${BUILD_PARALLEL}" + + - name: Show ccache stats + run: ccache --show-stats + + - name: Run IPC unit tests + run: | + LD_LIBRARY_PATH="depends/${{ steps.host.outputs.host }}/lib" \ + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_unit_tests.sh "${TSAN_UNIT_TEST_RUNS}" + + - name: Run IPC functional tests + env: + CI_FAILFAST_TEST_LEAVE_DANGLING: 1 + run: | + LD_LIBRARY_PATH="depends/${{ steps.host.outputs.host }}/lib" \ + src/ipc/libmultiprocess/ci/scripts/run_bitcoin_core_functional_tests.sh "${TSAN_FUNCTIONAL_TEST_RUNS}" "${PARALLEL}" "10" + + - name: Save ccache + uses: actions/cache/save@v4 + if: github.ref == 'refs/heads/master' || steps.ccache-restore.outputs.cache-hit != 'true' + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-TSan-${{ github.ref }}-${{ github.sha }} From 3dd6d57047aa9c96c5c41cf6a1d1c51499711355 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Thu, 12 Mar 2026 21:21:07 +0100 Subject: [PATCH 9/9] ci: set Bitcoin Core CI test repetition --- .github/workflows/bitcoin-core-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bitcoin-core-ci.yml b/.github/workflows/bitcoin-core-ci.yml index 75fe947..fca453e 100644 --- a/.github/workflows/bitcoin-core-ci.yml +++ b/.github/workflows/bitcoin-core-ci.yml @@ -20,15 +20,15 @@ env: BITCOIN_REPO: bitcoin/bitcoin LLVM_VERSION: 22 LIBCXX_DIR: /tmp/libcxx-build/ - ASAN_UBSAN_UNIT_TEST_RUNS: 1 - ASAN_UBSAN_FUNCTIONAL_TEST_RUNS: 1 - ASAN_UBSAN_NPROC_MULTIPLIER: 1 - MACOS_UNIT_TEST_RUNS: 1 - MACOS_FUNCTIONAL_TEST_RUNS: 1 - MACOS_NPROC_MULTIPLIER: 1 - TSAN_UNIT_TEST_RUNS: 1 - TSAN_FUNCTIONAL_TEST_RUNS: 1 - TSAN_NPROC_MULTIPLIER: 1 + ASAN_UBSAN_UNIT_TEST_RUNS: 15 + ASAN_UBSAN_FUNCTIONAL_TEST_RUNS: 300 + ASAN_UBSAN_NPROC_MULTIPLIER: 10 + MACOS_UNIT_TEST_RUNS: 50 + MACOS_FUNCTIONAL_TEST_RUNS: 200 + MACOS_NPROC_MULTIPLIER: 5 + TSAN_UNIT_TEST_RUNS: 8 + TSAN_FUNCTIONAL_TEST_RUNS: 300 + TSAN_NPROC_MULTIPLIER: 10 jobs: bitcoin-core: