From 34fce1b8b51fa3b25457e874a2367fd478c6be76 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Fri, 31 Jul 2026 06:46:44 +0000 Subject: [PATCH 1/2] Explain figure rendering failures caused by kernel subshells JupyterLab >= 4.4 routes widget messages over kernel subshells, which ipykernel >= 7 services on their own threads. Drawing a live canvas then races with figure creation during cell execution, and Matplotlib is not thread-safe. The resulting mathtext parse error gives users no hint of the cause or of the available workarounds. When laying out a canvas fails while subshells are active, chain the original error into one that names the cause and both workarounds, and document them alongside the installation instructions. Co-Authored-By: Claude Opus 5 --- docs/getting-started/installation.md | 16 +++ src/plopp/backends/matplotlib/canvas.py | 13 ++- src/plopp/backends/matplotlib/utils.py | 39 +++++++ tests/backends/matplotlib/mpl_utils_test.py | 108 ++++++++++++++++++++ 4 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 tests/backends/matplotlib/mpl_utils_test.py diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 496eba90..36eb9a9a 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -19,3 +19,19 @@ conda install -c conda-forge plopp ``` ```` ````` + +## Interactive figures in JupyterLab + +Interactive figures (`%matplotlib widget`) can fail to render in JupyterLab 4.4 and later, +with math text parse errors, blank figures, or kernel crashes. +JupyterLab routes widget messages over kernel subshells, which ipykernel 7 and later +service on their own threads. +Drawing a live canvas then races with figure creation during cell execution, +and Matplotlib is not thread-safe +(see [ipympl#610](https://github.com/matplotlib/ipympl/issues/610)). + +Until this is fixed upstream, either + +- set `commsOverSubshells` to `disabled` in the JupyterLab settings editor and restart + JupyterLab (the setting only applies to newly connected kernels), or +- install `ipykernel<7`. diff --git a/src/plopp/backends/matplotlib/canvas.py b/src/plopp/backends/matplotlib/canvas.py index d23b05fd..952d6a5d 100644 --- a/src/plopp/backends/matplotlib/canvas.py +++ b/src/plopp/backends/matplotlib/canvas.py @@ -15,7 +15,14 @@ from ...core.utils import maybe_variable_to_number, scalar_to_string from ...graphics.bbox import BoundingBox from ...utils import parse_mutually_exclusive -from .utils import fig_to_bytes, is_sphinx_build, make_figure, make_legend +from .utils import ( + SUBSHELL_CONCURRENCY_MESSAGE, + fig_to_bytes, + is_sphinx_build, + make_figure, + make_legend, + subshells_in_use, +) def _cursor_value_to_variable(x: float, dtype: sc.DType, unit: str) -> sc.Variable: @@ -342,6 +349,10 @@ def to_widget(self): self.fig.tight_layout() except RuntimeError: pass + except Exception as e: + if not subshells_in_use(): + raise + raise RuntimeError(SUBSHELL_CONCURRENCY_MESSAGE) from e # The Matplotlib canvas tries to fill the entire width of the output cell, # which can add unnecessary whitespace between it and other widgets. To # prevent this, we wrap the canvas in a VBox, which seems to help. diff --git a/src/plopp/backends/matplotlib/utils.py b/src/plopp/backends/matplotlib/utils.py index 6dbd6760..b258be95 100644 --- a/src/plopp/backends/matplotlib/utils.py +++ b/src/plopp/backends/matplotlib/utils.py @@ -125,6 +125,45 @@ def is_sphinx_build() -> bool: return meta.get("scipp_sphinx_build", False) +SUBSHELL_CONCURRENCY_MESSAGE = """Failed to render the figure because Matplotlib \ +state was modified concurrently from another thread. + +JupyterLab >= 4.4 routes widget messages over kernel subshells, which ipykernel >= 7 \ +services on their own threads. Drawing a live canvas thus races with figure creation \ +during cell execution, and Matplotlib is not thread-safe. Symptoms include math text \ +parse errors (as chained above), blank figures, and kernel crashes. + +Workarounds: +- In the JupyterLab settings editor, set 'commsOverSubshells' to 'disabled' and \ +restart JupyterLab (the setting only applies to newly connected kernels), or +- install ipykernel < 7. + +See https://github.com/matplotlib/ipympl/issues/610 for details.""" + + +def subshells_in_use() -> bool: + """ + Return ``True`` if the Jupyter kernel is servicing messages on subshell threads. + + Subshells run concurrently with cell execution, which is unsafe for Matplotlib + figures that are alive in the notebook (see ``SUBSHELL_CONCURRENCY_MESSAGE``). + This is a best-effort diagnostic used to explain rendering failures, hence any + error while inspecting the kernel means we simply cannot tell. + """ + try: + from ipykernel.kernelapp import IPKernelApp + + if not IPKernelApp.initialized(): + return False + kernel = IPKernelApp.instance().kernel + # ipykernel exposes no public API to ask whether subshells are supported. + if not getattr(kernel, '_supports_kernel_subshells', False): + return False + return bool(kernel.shell_channel_thread.manager.list_subshell()) + except Exception: + return False + + def parse_dicts_in_kwargs(kwargs, name): out = {} for key, value in kwargs.items(): diff --git a/tests/backends/matplotlib/mpl_utils_test.py b/tests/backends/matplotlib/mpl_utils_test.py new file mode 100644 index 00000000..7f7f3998 --- /dev/null +++ b/tests/backends/matplotlib/mpl_utils_test.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +import ipykernel.kernelapp +import pytest + +from plopp.backends.matplotlib.canvas import Canvas +from plopp.backends.matplotlib.utils import subshells_in_use + + +class FakeSubshellManager: + def __init__(self, subshells: list[str]): + self._subshells = subshells + + def list_subshell(self) -> list[str]: + return self._subshells + + +class FakeShellChannelThread: + def __init__(self, subshells: list[str]): + self.manager = FakeSubshellManager(subshells) + + +class FakeKernel: + def __init__(self, supports_subshells: bool, subshells: list[str]): + self._supports_kernel_subshells = supports_subshells + self.shell_channel_thread = FakeShellChannelThread(subshells) + + +class FakeKernelApp: + """Stub for ``IPKernelApp``, which is both a singleton and its own factory.""" + + def __init__(self, kernel: object | None): + self.kernel = kernel + + def initialized(self) -> bool: + return self.kernel is not None + + def instance(self) -> 'FakeKernelApp': + return self + + +@pytest.fixture +def fake_kernel_app(monkeypatch): + def set_kernel(kernel: object | None): + monkeypatch.setattr( + ipykernel.kernelapp, 'IPKernelApp', FakeKernelApp(kernel=kernel) + ) + + return set_kernel + + +def test_subshells_in_use_no_kernel(): + assert not subshells_in_use() + + +def test_subshells_in_use_kernel_not_initialized(fake_kernel_app): + fake_kernel_app(None) + assert not subshells_in_use() + + +def test_subshells_in_use_kernel_without_subshell_support(fake_kernel_app): + fake_kernel_app(FakeKernel(supports_subshells=False, subshells=[])) + assert not subshells_in_use() + + +def test_subshells_in_use_no_subshells_created(fake_kernel_app): + fake_kernel_app(FakeKernel(supports_subshells=True, subshells=[])) + assert not subshells_in_use() + + +def test_subshells_in_use_with_subshells(fake_kernel_app): + fake_kernel_app(FakeKernel(supports_subshells=True, subshells=['abcd-1234'])) + assert subshells_in_use() + + +def test_subshells_in_use_unexpected_kernel_api(fake_kernel_app): + class KernelWithoutShellChannelThread: + _supports_kernel_subshells = True + + fake_kernel_app(KernelWithoutShellChannelThread()) + assert not subshells_in_use() + + +def broken_canvas() -> Canvas: + """Canvas whose layout fails, as it does when a subshell corrupts Matplotlib.""" + + def tight_layout(): + raise ValueError('mathtext ParseException') + + canvas = Canvas() + canvas.fig.tight_layout = tight_layout + return canvas + + +@pytest.mark.usefixtures('_use_ipympl') +def test_to_widget_explains_concurrent_subshells(fake_kernel_app): + fake_kernel_app(FakeKernel(supports_subshells=True, subshells=['abcd-1234'])) + with pytest.raises(RuntimeError, match='commsOverSubshells') as info: + broken_canvas().to_widget() + assert isinstance(info.value.__cause__, ValueError) + + +@pytest.mark.usefixtures('_use_ipympl') +def test_to_widget_reraises_when_no_subshells(fake_kernel_app): + fake_kernel_app(FakeKernel(supports_subshells=True, subshells=[])) + with pytest.raises(ValueError, match='mathtext ParseException'): + broken_canvas().to_widget() From 88bf25dc71c4dc8154fa43aef12d69d5ce25c376 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 3 Aug 2026 03:24:32 +0000 Subject: [PATCH 2/2] Reduce to documentation of subshell concurrency workarounds The runtime detection hooked the math text ParseException raised during tight_layout. That symptom is the one an ipympl-side parser lock removes; what survives the lock -- blank canvases and kernel crashes -- raises no exception, so the detection would catch nothing once ipympl releases. The workarounds are unchanged and remain the only complete fix, so keep them documented next to the installation instructions. --- docs/getting-started/installation.md | 5 +- src/plopp/backends/matplotlib/canvas.py | 13 +-- src/plopp/backends/matplotlib/utils.py | 39 ------- tests/backends/matplotlib/mpl_utils_test.py | 108 -------------------- 4 files changed, 5 insertions(+), 160 deletions(-) delete mode 100644 tests/backends/matplotlib/mpl_utils_test.py diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 36eb9a9a..fc24361d 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -30,7 +30,10 @@ Drawing a live canvas then races with figure creation during cell execution, and Matplotlib is not thread-safe (see [ipympl#610](https://github.com/matplotlib/ipympl/issues/610)). -Until this is fixed upstream, either +A lock added in ipympl removes the math text parse errors, but the blank figures and +crashes remain, since the canvas frame and resize handlers still run concurrently with +drawing. +Until that is addressed, either - set `commsOverSubshells` to `disabled` in the JupyterLab settings editor and restart JupyterLab (the setting only applies to newly connected kernels), or diff --git a/src/plopp/backends/matplotlib/canvas.py b/src/plopp/backends/matplotlib/canvas.py index 952d6a5d..d23b05fd 100644 --- a/src/plopp/backends/matplotlib/canvas.py +++ b/src/plopp/backends/matplotlib/canvas.py @@ -15,14 +15,7 @@ from ...core.utils import maybe_variable_to_number, scalar_to_string from ...graphics.bbox import BoundingBox from ...utils import parse_mutually_exclusive -from .utils import ( - SUBSHELL_CONCURRENCY_MESSAGE, - fig_to_bytes, - is_sphinx_build, - make_figure, - make_legend, - subshells_in_use, -) +from .utils import fig_to_bytes, is_sphinx_build, make_figure, make_legend def _cursor_value_to_variable(x: float, dtype: sc.DType, unit: str) -> sc.Variable: @@ -349,10 +342,6 @@ def to_widget(self): self.fig.tight_layout() except RuntimeError: pass - except Exception as e: - if not subshells_in_use(): - raise - raise RuntimeError(SUBSHELL_CONCURRENCY_MESSAGE) from e # The Matplotlib canvas tries to fill the entire width of the output cell, # which can add unnecessary whitespace between it and other widgets. To # prevent this, we wrap the canvas in a VBox, which seems to help. diff --git a/src/plopp/backends/matplotlib/utils.py b/src/plopp/backends/matplotlib/utils.py index b258be95..6dbd6760 100644 --- a/src/plopp/backends/matplotlib/utils.py +++ b/src/plopp/backends/matplotlib/utils.py @@ -125,45 +125,6 @@ def is_sphinx_build() -> bool: return meta.get("scipp_sphinx_build", False) -SUBSHELL_CONCURRENCY_MESSAGE = """Failed to render the figure because Matplotlib \ -state was modified concurrently from another thread. - -JupyterLab >= 4.4 routes widget messages over kernel subshells, which ipykernel >= 7 \ -services on their own threads. Drawing a live canvas thus races with figure creation \ -during cell execution, and Matplotlib is not thread-safe. Symptoms include math text \ -parse errors (as chained above), blank figures, and kernel crashes. - -Workarounds: -- In the JupyterLab settings editor, set 'commsOverSubshells' to 'disabled' and \ -restart JupyterLab (the setting only applies to newly connected kernels), or -- install ipykernel < 7. - -See https://github.com/matplotlib/ipympl/issues/610 for details.""" - - -def subshells_in_use() -> bool: - """ - Return ``True`` if the Jupyter kernel is servicing messages on subshell threads. - - Subshells run concurrently with cell execution, which is unsafe for Matplotlib - figures that are alive in the notebook (see ``SUBSHELL_CONCURRENCY_MESSAGE``). - This is a best-effort diagnostic used to explain rendering failures, hence any - error while inspecting the kernel means we simply cannot tell. - """ - try: - from ipykernel.kernelapp import IPKernelApp - - if not IPKernelApp.initialized(): - return False - kernel = IPKernelApp.instance().kernel - # ipykernel exposes no public API to ask whether subshells are supported. - if not getattr(kernel, '_supports_kernel_subshells', False): - return False - return bool(kernel.shell_channel_thread.manager.list_subshell()) - except Exception: - return False - - def parse_dicts_in_kwargs(kwargs, name): out = {} for key, value in kwargs.items(): diff --git a/tests/backends/matplotlib/mpl_utils_test.py b/tests/backends/matplotlib/mpl_utils_test.py deleted file mode 100644 index 7f7f3998..00000000 --- a/tests/backends/matplotlib/mpl_utils_test.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - -import ipykernel.kernelapp -import pytest - -from plopp.backends.matplotlib.canvas import Canvas -from plopp.backends.matplotlib.utils import subshells_in_use - - -class FakeSubshellManager: - def __init__(self, subshells: list[str]): - self._subshells = subshells - - def list_subshell(self) -> list[str]: - return self._subshells - - -class FakeShellChannelThread: - def __init__(self, subshells: list[str]): - self.manager = FakeSubshellManager(subshells) - - -class FakeKernel: - def __init__(self, supports_subshells: bool, subshells: list[str]): - self._supports_kernel_subshells = supports_subshells - self.shell_channel_thread = FakeShellChannelThread(subshells) - - -class FakeKernelApp: - """Stub for ``IPKernelApp``, which is both a singleton and its own factory.""" - - def __init__(self, kernel: object | None): - self.kernel = kernel - - def initialized(self) -> bool: - return self.kernel is not None - - def instance(self) -> 'FakeKernelApp': - return self - - -@pytest.fixture -def fake_kernel_app(monkeypatch): - def set_kernel(kernel: object | None): - monkeypatch.setattr( - ipykernel.kernelapp, 'IPKernelApp', FakeKernelApp(kernel=kernel) - ) - - return set_kernel - - -def test_subshells_in_use_no_kernel(): - assert not subshells_in_use() - - -def test_subshells_in_use_kernel_not_initialized(fake_kernel_app): - fake_kernel_app(None) - assert not subshells_in_use() - - -def test_subshells_in_use_kernel_without_subshell_support(fake_kernel_app): - fake_kernel_app(FakeKernel(supports_subshells=False, subshells=[])) - assert not subshells_in_use() - - -def test_subshells_in_use_no_subshells_created(fake_kernel_app): - fake_kernel_app(FakeKernel(supports_subshells=True, subshells=[])) - assert not subshells_in_use() - - -def test_subshells_in_use_with_subshells(fake_kernel_app): - fake_kernel_app(FakeKernel(supports_subshells=True, subshells=['abcd-1234'])) - assert subshells_in_use() - - -def test_subshells_in_use_unexpected_kernel_api(fake_kernel_app): - class KernelWithoutShellChannelThread: - _supports_kernel_subshells = True - - fake_kernel_app(KernelWithoutShellChannelThread()) - assert not subshells_in_use() - - -def broken_canvas() -> Canvas: - """Canvas whose layout fails, as it does when a subshell corrupts Matplotlib.""" - - def tight_layout(): - raise ValueError('mathtext ParseException') - - canvas = Canvas() - canvas.fig.tight_layout = tight_layout - return canvas - - -@pytest.mark.usefixtures('_use_ipympl') -def test_to_widget_explains_concurrent_subshells(fake_kernel_app): - fake_kernel_app(FakeKernel(supports_subshells=True, subshells=['abcd-1234'])) - with pytest.raises(RuntimeError, match='commsOverSubshells') as info: - broken_canvas().to_widget() - assert isinstance(info.value.__cause__, ValueError) - - -@pytest.mark.usefixtures('_use_ipympl') -def test_to_widget_reraises_when_no_subshells(fake_kernel_app): - fake_kernel_app(FakeKernel(supports_subshells=True, subshells=[])) - with pytest.raises(ValueError, match='mathtext ParseException'): - broken_canvas().to_widget()