From 68a8b391e66567b9e2d90c6451d18c8bbd2390b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 22:12:16 +0000 Subject: [PATCH] fix(test): seed the flaky interpolator covariance tests test_variable_and_constant is non-deterministic and fails ~1 CI run in 60. It is the sole reason the 2026.8.2.1 live release failed (1641 passed, 1 failed, assert 30.121646313498022 == 25.0 +/- 5). Measured before changing anything, over 2000 runs under the same conditions CI uses (the autouse limit_maxcall fixture caps the search at maxcall=1): 33/2000 = 1.65% miss abs=5.0 (95% CI 1.18-2.31%), recovered value mean 25.24, std 2.05, range 17.82-32.04. The observed 30.12 sits well inside that distribution. The tolerance is not the problem. With a full, untruncated search the same test recovers 25.0496 +/- 0.0039 - 500x tighter. limit_maxcall caps every search here at one likelihood call, so the recovered value is an unconverged draw whose spread is set entirely by the search's randomness. With all seeds fixed the interpolator's inputs (x, y, inverse covariance) are bit-identical; all the variance is inside search.fit. Three generators feed it, so seeding any one is insufficient: 1. numpy.random - the test body's two np.random.random() calls. Seeding only these leaves std at 1.95 and still fails. Not dominant. 2. the stdlib random module - initializer.py draws initial unit values with random.uniform. 3. dynesty's rstate - defaults to Generator(PCG64(None)), OS entropy, reachable from neither of the above. Dominant. Seeding all three from one autouse fixture, restoring global generator state on teardown so nothing leaks into later tests in the same process. This also fixes a worse sibling: test_single_variable makes no random call of its own, yet produced 500 distinct values over 500 runs and missed its abs=2.0 tolerance 3.6% of the time. It depends on source (3) alone, so seeding np.random would never have fixed it. Ruled out: the scipy.linalg.LinAlgError guards on test_interpolate and friends are not this nondeterminism in disguise. The interpolator fixture uses no RNG; its covariance matrix is bit-identical across builds and rank-deficient (rank 6 of 9, cond 1.3e17). Those guards cover inverting a singular matrix - deterministic input, platform-LAPACK dependent. Recovered values are now bit-identical across 8 invocations and in both selection contexts: test_single_variable 24.577316739353 (dev 0.42 vs tol 2.0), test_variable_and_constant 22.718814166116 (dev 2.28 vs tol 5.0). Test-only; no library source or public API change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016vdGab4C5rWumTCXkWHPyh --- test_autofit/interpolator/test_covariance.py | 55 +++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/test_autofit/interpolator/test_covariance.py b/test_autofit/interpolator/test_covariance.py index 6a1bfc5ec..18b4bfe8a 100644 --- a/test_autofit/interpolator/test_covariance.py +++ b/test_autofit/interpolator/test_covariance.py @@ -1,4 +1,5 @@ import logging +import random from unittest.mock import patch import pytest @@ -10,6 +11,8 @@ import autofit as af from autofit.non_linear.search.nest.dynesty.search.static import DynestyStatic +SEED = 20260802 + @pytest.fixture(autouse=True) def do_remove_output(output_directory, remove_output): @@ -54,6 +57,53 @@ def patched_init(self, *args, **kwargs): monkeypatch.setattr(DynestyStatic, "__init__", patched_init) +@pytest.fixture(autouse=True) +def seed_search_randomness(monkeypatch): + """ + Make the Dynesty searches these tests run reproducible. + + `limit_maxcall` above caps each search at a single likelihood call so this + module stays fast, which leaves the recovered value dominated by the + sampler's randomness rather than by convergence. Unseeded, the numerical + recovery assertions are therefore coin flips: measured over 2000 runs, + `test_variable_and_constant` missed `abs=5.0` 1.65% of the time (95% CI + 1.18-2.31%) and `test_single_variable` missed `abs=2.0` 3.6% of the time. + The 1.65% is what killed the 2026.8.2.1 live release. + + Three independent generators feed that result, so seeding any one of them + is not enough: + + * `numpy.random`, used by `test_variable_and_constant` to build its samples; + * the stdlib `random` module, used by `autofit.non_linear.initializer` to + draw the search's initial unit values; + * dynesty's own `rstate`, which defaults to + `numpy.random.Generator(PCG64(None))` — seeded from OS entropy and + reachable from neither of the above. This is the dominant term, and the + only one `test_single_variable` (which makes no random call of its own) + depends on at all. + + Global generator state is restored on teardown so the seeding cannot leak + into whatever runs next in the same process. + """ + import dynesty.dynesty + + monkeypatch.setattr( + dynesty.dynesty, + "get_random_generator", + lambda seed=None: np.random.default_rng(SEED if seed is None else seed), + ) + + random_state = random.getstate() + numpy_state = np.random.get_state() + random.seed(SEED) + np.random.seed(SEED) + try: + yield + finally: + random.setstate(random_state) + np.random.set_state(numpy_state) + + def test_interpolate(interpolator): try: assert isinstance(interpolator[interpolator.t == 0.5].gaussian.centre, float) @@ -120,6 +170,7 @@ def test_single_variable(): def test_variable_and_constant(): + rng = np.random.default_rng(SEED) samples_list = [ af.SamplesPDF( model=af.Collection( @@ -133,8 +184,8 @@ def test_variable_and_constant(): log_prior=1.0, weight=1.0, kwargs={ - ("v",): value + 0.1 * (1 - np.random.random()), - ("x",): 0.5 * (1 - +np.random.random()), + ("v",): value + 0.1 * (1 - rng.random()), + ("x",): 0.5 * (1 - +rng.random()), }, ) for _ in range(50)