|
4 | 4 | modules. |
5 | 5 | """ |
6 | 6 |
|
7 | | -import numpy as np |
| 7 | +import numpy as np |
8 | 8 | import time |
| 9 | +import os |
| 10 | +import sys |
9 | 11 |
|
10 | 12 | # Supports both pypar and mpi4py |
11 | 13 | __PYPAR__ = False |
@@ -46,14 +48,53 @@ def am_i_the_master(): |
46 | 48 | else: |
47 | 49 | return True |
48 | 50 |
|
49 | | -def pprint(*argv): |
| 51 | +def _force_stdout_blocking(): |
| 52 | + """ |
| 53 | + Make sure the standard output is in blocking mode. |
| 54 | +
|
| 55 | + MPI launchers (mpirun, srun, ...) frequently attach the standard output |
| 56 | + of the processes to a pipe that is opened in *non-blocking* mode. When a |
| 57 | + large message fills the pipe buffer, a write on a non-blocking descriptor |
| 58 | + raises ``BlockingIOError`` ([Errno 11]) instead of waiting for the buffer |
| 59 | + to drain. This used to crash a parallel minimization while printing the |
| 60 | + table of imaginary frequencies (see issue #196). |
| 61 | +
|
| 62 | + Restoring the blocking mode makes the write wait for the reader to consume |
| 63 | + the buffer, which is the expected behaviour for log output. |
| 64 | + """ |
| 65 | + if not hasattr(os, "set_blocking"): |
| 66 | + return |
| 67 | + try: |
| 68 | + fd = sys.stdout.fileno() |
| 69 | + except (AttributeError, OSError, ValueError): |
| 70 | + # stdout has been replaced by an object without a real descriptor |
| 71 | + return |
| 72 | + try: |
| 73 | + os.set_blocking(fd, True) |
| 74 | + except (OSError, ValueError): |
| 75 | + pass |
| 76 | + |
| 77 | + |
| 78 | +def pprint(*argv, **kwargs): |
50 | 79 | """ |
51 | 80 | PARALLEL PRINTING |
52 | 81 | ================= |
53 | 82 |
|
54 | | - This will print on stdout only once in parallel execution of the code |
| 83 | + This will print on stdout only once in parallel execution of the code. |
| 84 | +
|
| 85 | + It is robust against a standard output opened in non-blocking mode, which |
| 86 | + is a frequent source of ``BlockingIOError`` when running under MPI (see |
| 87 | + issue #196): a log line must never abort a running calculation. |
55 | 88 | """ |
56 | 89 | #print("pypar:", __PYPAR__) |
57 | 90 | #print("mpi4py:", __MPI4PY__) |
58 | | - if am_i_the_master(): |
59 | | - print(*argv) |
| 91 | + if not am_i_the_master(): |
| 92 | + return |
| 93 | + |
| 94 | + _force_stdout_blocking() |
| 95 | + try: |
| 96 | + print(*argv, **kwargs) |
| 97 | + except BlockingIOError: |
| 98 | + # The blocking mode could not be enforced (e.g. os.set_blocking is |
| 99 | + # not available or failed). Never let a print crash the calculation. |
| 100 | + pass |
0 commit comments