From d65a3b15d75a2ccf72a76572bec29cf712e14ebc Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 14 May 2026 14:32:07 -0400 Subject: [PATCH 1/3] Catch python exceptions to prevent interacting with destroyed python objects --- src/NGen.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 0d1a4161bb..a757bc8a1b 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -36,6 +36,7 @@ #if NGEN_WITH_PYTHON #include "python/InterpreterUtil.hpp" #include +#include #endif // NGEN_WITH_PYTHON #if NGEN_WITH_MPI @@ -877,6 +878,19 @@ int main(int argc, char* argv[]) { auto interp = utils::ngenPy::InterpreterUtil::getInstance(); try { result = run_ngen(argc, argv, mpi_num_procs, mpi_rank); + + } catch (pybind11::error_already_set &e) { + // If the error comes from python, + // we cannot rethrow the execption after destroying the interpreter (doing so sometimes caused MPI runs to hang) + // First, copy the python error message so it an be reraised as a runtime_error + std::string error_msg = std::string("Uncaught python error: ") + e.what(); + // direct the exception's full stack trace to be printed to stderr + e.discard_as_unraisable("Passing python error before tearing down interpreter."); + // destroy the interpreter to let python atexit actions trigger + interp.reset(); + // throw the copied error to ensure MPI acknowledges the termination + throw std::runtime_error(error_msg); + } catch (...) { // If any uncaught exception happens, // explictly destroy the interpreter to ensure any @@ -892,7 +906,7 @@ int main(int argc, char* argv[]) { // this is needed if any python atexit registered functions would interact with MPI MPI_Barrier(MPI_COMM_WORLD); #endif // NGEN_WITH_MPI -#else +#else // not NGEN_WITH_PYTHON result = run_ngen(argc, argv, mpi_num_procs, mpi_rank); #endif // NGEN_WITH_PYTHON From 4f8dcda23c2d4303835633151caf54e4c846f70d Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 15 May 2026 14:16:12 -0400 Subject: [PATCH 2/3] Remove logging exception through python --- src/NGen.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index a757bc8a1b..d85c414a3f 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -884,8 +884,6 @@ int main(int argc, char* argv[]) { // we cannot rethrow the execption after destroying the interpreter (doing so sometimes caused MPI runs to hang) // First, copy the python error message so it an be reraised as a runtime_error std::string error_msg = std::string("Uncaught python error: ") + e.what(); - // direct the exception's full stack trace to be printed to stderr - e.discard_as_unraisable("Passing python error before tearing down interpreter."); // destroy the interpreter to let python atexit actions trigger interp.reset(); // throw the copied error to ensure MPI acknowledges the termination From b59b3164672378b5bb69df899d9d37d6b02faa79 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 7 Jul 2026 11:23:06 -0400 Subject: [PATCH 3/3] Log uncaught python exception --- src/NGen.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/NGen.cpp b/src/NGen.cpp index d85c414a3f..71de0b6f27 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -884,6 +884,9 @@ int main(int argc, char* argv[]) { // we cannot rethrow the execption after destroying the interpreter (doing so sometimes caused MPI runs to hang) // First, copy the python error message so it an be reraised as a runtime_error std::string error_msg = std::string("Uncaught python error: ") + e.what(); + // Log as a fatal error since it will end the program. + // This may result in double logging, but better to make sure it gets logged. + LOG(LogLevel::FATAL, error_msg); // destroy the interpreter to let python atexit actions trigger interp.reset(); // throw the copied error to ensure MPI acknowledges the termination