From b528ca2f29498a84a213a9d5b0a161d8d100294b Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Fri, 16 Jan 2026 15:57:53 +0100 Subject: [PATCH 01/15] Move some tests to use skipped smaller data --- src/openfe_analysis/tests/test_reader.py | 45 +++++++++---------- .../tests/test_transformations.py | 16 ++++--- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/openfe_analysis/tests/test_reader.py b/src/openfe_analysis/tests/test_reader.py index 6cef163..649186d 100644 --- a/src/openfe_analysis/tests/test_reader.py +++ b/src/openfe_analysis/tests/test_reader.py @@ -106,47 +106,44 @@ def test_universe_creation(simulation_nc, hybrid_system_pdb): assert_allclose(u.dimensions, [82.191055, 82.191055, 82.191055, 90.0, 90.0, 90.0]) -def test_universe_from_nc_file(simulation_nc, hybrid_system_pdb): - ds = nc.Dataset(simulation_nc) +def test_universe_from_nc_file(simulation_skipped_nc, hybrid_system_skipped_pdb): + ds = nc.Dataset(simulation_skipped_nc) - with pytest.warns(UserWarning, match="This is an older NetCDF file that"): - u = mda.Universe(hybrid_system_pdb, ds, format="MultiStateReporter", state_id=0) + u = mda.Universe(hybrid_system_skipped_pdb, ds, format="MultiStateReporter", state_id=0) assert u - assert len(u.atoms) == 4782 - assert len(u.trajectory) == 501 - assert u.trajectory.dt == pytest.approx(1.0) - + assert len(u.atoms) == 4762 + assert len(u.trajectory) == 6 + assert u.trajectory.dt == pytest.approx(100.0) -def test_universe_creation_noconversion(simulation_nc, hybrid_system_pdb): - with pytest.warns(UserWarning, match="This is an older NetCDF file that"): - u = mda.Universe( - hybrid_system_pdb, simulation_nc, format=FEReader, state_id=0, convert_units=False - ) +def test_universe_creation_noconversion(simulation_skipped_nc, hybrid_system_skipped_pdb): + u = mda.Universe( + hybrid_system_skipped_pdb, simulation_skipped_nc, format=FEReader, state_id=0, convert_units=False + ) + assert u.trajectory.ts.frame == 0 assert_allclose( u.atoms[:3].positions, np.array( [ - [6.51474, -1.7640617, 8.406607], - [6.641961, -1.8410535, 8.433087], - [6.71369, -1.8112476, 8.533738], + [2.778386, 2.733918, 6.116591], + [2.836767, 2.600875, 6.174912], + [2.917513, 2.604454, 6.273793], ] ), + atol=1e-6, ) -def test_fereader_negative_state(simulation_nc, hybrid_system_pdb): - with pytest.warns(UserWarning, match="This is an older NetCDF file that"): - u = mda.Universe(hybrid_system_pdb, simulation_nc, format=FEReader, state_id=-1) +def test_fereader_negative_state(simulation_skipped_nc, hybrid_system_skipped_pdb): + u = mda.Universe(hybrid_system_skipped_pdb, simulation_skipped_nc, format=FEReader, state_id=-1) assert u.trajectory._state_id == 10 assert u.trajectory._replica_id is None -def test_fereader_negative_replica(simulation_nc, hybrid_system_pdb): - with pytest.warns(UserWarning, match="This is an older NetCDF file that"): - u = mda.Universe(hybrid_system_pdb, simulation_nc, format=FEReader, replica_id=-2) +def test_fereader_negative_replica(simulation_skipped_nc, hybrid_system_skipped_pdb): + u = mda.Universe(hybrid_system_skipped_pdb, simulation_skipped_nc, format=FEReader, replica_id=-2) assert u.trajectory._state_id is None assert u.trajectory._replica_id == 9 @@ -154,10 +151,10 @@ def test_fereader_negative_replica(simulation_nc, hybrid_system_pdb): @pytest.mark.parametrize("rep_id, state_id", [[None, None], [1, 1]]) @pytest.mark.flaky(reruns=3) -def test_fereader_replica_state_id_error(simulation_nc, hybrid_system_pdb, rep_id, state_id): +def test_fereader_replica_state_id_error(simulation_skipped_nc, hybrid_system_skipped_pdb, rep_id, state_id): with pytest.raises(ValueError, match="Specify one and only one"): _ = mda.Universe( - hybrid_system_pdb, simulation_nc, format=FEReader, state_id=state_id, replica_id=rep_id + hybrid_system_skipped_pdb, simulation_skipped_nc, format=FEReader, state_id=state_id, replica_id=rep_id ) diff --git a/src/openfe_analysis/tests/test_transformations.py b/src/openfe_analysis/tests/test_transformations.py index 96d4a0f..c31f8b1 100644 --- a/src/openfe_analysis/tests/test_transformations.py +++ b/src/openfe_analysis/tests/test_transformations.py @@ -12,10 +12,10 @@ @pytest.fixture -def universe(hybrid_system_pdb, simulation_nc): +def universe(hybrid_system_skipped_pdb, simulation_skipped_nc): return mda.Universe( - hybrid_system_pdb, - simulation_nc, + hybrid_system_skipped_pdb, + simulation_skipped_nc, format="MultiStateReporter", state_id=0, ) @@ -31,11 +31,17 @@ def test_minimiser(universe): d = mda.lib.distances.calc_bonds(prot.center_of_mass(), lig.center_of_mass()) # in the raw trajectory this is ~71 A as they're in diff images # accounting for pbc should result in ~11.10 - assert d == pytest.approx(11.10, abs=0.01) + assert d == pytest.approx(11.78, abs=0.01) @pytest.mark.flaky(reruns=3) -def test_nojump(universe): +def test_nojump(hybrid_system_pdb, simulation_nc): + universe = mda.Universe( + hybrid_system_pdb, + simulation_nc, + format="MultiStateReporter", + state_id=0, + ) # find frame where protein would teleport across boundary and check it prot = universe.select_atoms("protein and name CA") From a477bc1ca49298e4df485760351157c55c8fff26 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Fri, 16 Jan 2026 16:36:53 +0100 Subject: [PATCH 02/15] Test out zenodo dealings --- src/openfe_analysis/tests/conftest.py | 54 ++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/src/openfe_analysis/tests/conftest.py b/src/openfe_analysis/tests/conftest.py index 5fa2db4..b290659 100644 --- a/src/openfe_analysis/tests/conftest.py +++ b/src/openfe_analysis/tests/conftest.py @@ -3,8 +3,10 @@ import pathlib import pooch import pytest +from filelock import FileLock POOCH_CACHE = pooch.os_cache("openfe_analysis") +POOCH_CACHE.mkdir(parents=True, exist_ok=True) ZENODO_RBFE_DATA = pooch.create( path=POOCH_CACHE, base_url="doi:10.5281/zenodo.17916322", @@ -14,17 +16,57 @@ }, ) +POOCH_LOCK = FileLock(str(POOCH_CACHE / "pooch.lock")) + +def _fetch_and_untar_once(archive_name: str, extracted_dir_name: str) -> pathlib.Path: + """ + Fetch and untar a Zenodo archive exactly once, safely across + multiple pytest workers or CI jobs. + + Returns the path to the extracted directory. + """ + untar_dir = ( + POOCH_CACHE + / f"{archive_name}.untar" + / extracted_dir_name + ) + + # Fast path: already fully extracted + if untar_dir.exists(): + return untar_dir + + # Slow path: lock + fetch + extract + with POOCH_LOCK: + # Another worker may have completed while we waited + if untar_dir.exists(): + return untar_dir + + ZENODO_RBFE_DATA.fetch( + archive_name, + processor=pooch.Untar(), + ) + + if not untar_dir.exists(): + raise RuntimeError( + f"Expected extracted directory not found: {untar_dir}" + ) + + return untar_dir + @pytest.fixture(scope="session") def rbfe_output_data_dir() -> pathlib.Path: - ZENODO_RBFE_DATA.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar()) - result_dir = pathlib.Path(POOCH_CACHE) / "openfe_analysis_simulation_output.tar.gz.untar/openfe_analysis_simulation_output/" - return result_dir + return _fetch_and_untar_once( + "openfe_analysis_simulation_output.tar.gz", + "openfe_analysis_simulation_output", + ) + @pytest.fixture(scope="session") def rbfe_skipped_data_dir() -> pathlib.Path: - ZENODO_RBFE_DATA.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) - result_dir = pathlib.Path(POOCH_CACHE) / "openfe_analysis_skipped.tar.gz.untar/openfe_analysis_skipped/" - return result_dir + return _fetch_and_untar_once( + "openfe_analysis_skipped.tar.gz", + "openfe_analysis_skipped", + ) @pytest.fixture(scope="session") def simulation_nc(rbfe_output_data_dir) -> pathlib.Path: From ad840824b5b7d05a9b9f9028f42564f5329a8754 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Fri, 16 Jan 2026 16:49:25 +0100 Subject: [PATCH 03/15] Try to improbe speed --- src/openfe_analysis/tests/conftest.py | 61 ++++++++++++--------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/src/openfe_analysis/tests/conftest.py b/src/openfe_analysis/tests/conftest.py index b290659..1a93ad7 100644 --- a/src/openfe_analysis/tests/conftest.py +++ b/src/openfe_analysis/tests/conftest.py @@ -7,6 +7,9 @@ POOCH_CACHE = pooch.os_cache("openfe_analysis") POOCH_CACHE.mkdir(parents=True, exist_ok=True) +LOCKFILE = POOCH_CACHE / "prepare.lock" +READY_FLAG = POOCH_CACHE / "data_ready.flag" + ZENODO_RBFE_DATA = pooch.create( path=POOCH_CACHE, base_url="doi:10.5281/zenodo.17916322", @@ -16,56 +19,44 @@ }, ) -POOCH_LOCK = FileLock(str(POOCH_CACHE / "pooch.lock")) - -def _fetch_and_untar_once(archive_name: str, extracted_dir_name: str) -> pathlib.Path: - """ - Fetch and untar a Zenodo archive exactly once, safely across - multiple pytest workers or CI jobs. +def _prepare_data(): + """Download and extract large test data once per machine.""" + if READY_FLAG.exists(): + return - Returns the path to the extracted directory. - """ - untar_dir = ( - POOCH_CACHE - / f"{archive_name}.untar" - / extracted_dir_name - ) - - # Fast path: already fully extracted - if untar_dir.exists(): - return untar_dir - - # Slow path: lock + fetch + extract - with POOCH_LOCK: - # Another worker may have completed while we waited - if untar_dir.exists(): - return untar_dir + with FileLock(str(LOCKFILE)): + if READY_FLAG.exists(): + return ZENODO_RBFE_DATA.fetch( - archive_name, + "openfe_analysis_simulation_output.tar.gz", + processor=pooch.Untar(), + ) + ZENODO_RBFE_DATA.fetch( + "openfe_analysis_skipped.tar.gz", processor=pooch.Untar(), ) - if not untar_dir.exists(): - raise RuntimeError( - f"Expected extracted directory not found: {untar_dir}" - ) + READY_FLAG.touch() - return untar_dir + +_prepare_data() @pytest.fixture(scope="session") def rbfe_output_data_dir() -> pathlib.Path: - return _fetch_and_untar_once( - "openfe_analysis_simulation_output.tar.gz", - "openfe_analysis_simulation_output", + return ( + POOCH_CACHE + / "openfe_analysis_simulation_output.tar.gz.untar" + / "openfe_analysis_simulation_output" ) @pytest.fixture(scope="session") def rbfe_skipped_data_dir() -> pathlib.Path: - return _fetch_and_untar_once( - "openfe_analysis_skipped.tar.gz", - "openfe_analysis_skipped", + return ( + POOCH_CACHE + / "openfe_analysis_skipped.tar.gz.untar" + / "openfe_analysis_skipped" ) @pytest.fixture(scope="session") From 8ba8087a08247a3a881e94e65d9260611c2687c9 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Fri, 16 Jan 2026 17:10:24 +0100 Subject: [PATCH 04/15] Try removing locking --- .github/workflows/ci.yaml | 41 ++++++++++++++++++++++++++ src/openfe_analysis/tests/conftest.py | 42 ++++++--------------------- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a5debbc..5163be8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -56,6 +56,47 @@ jobs: run: | python -Ic "import openfe_analysis; print(openfe_analysis.__version__)" + - name: Cache Pooch data + uses: actions/cache@v4 + with: + path: | + # linux cache location + ~/.cache/openfe_analysis + # osx cache location + ~/Library/Caches/openfe_analysis + # When files are added or changed in a pooch registry + # bump this key to create a new cache, for example if + # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 + key: pooch-${{ matrix.os }}-v1 + + - name: Pre-fetch Zenodo test data + run: | + python - <<'EOF' + import pooch + from pathlib import Path + + cache = Path(pooch.os_cache("openfe_analysis")) + cache.mkdir(parents=True, exist_ok=True) + + zenodo = pooch.create( + path=cache, + base_url="doi:10.5281/zenodo.17916322", + registry={ + "openfe_analysis_simulation_output.tar.gz": "md5:09752f2c4e5b7744d8afdee66dbd1414", + "openfe_analysis_skipped.tar.gz": "md5:3840d044299caacc4ccd50e6b22c0880", + }, + ) + + zenodo.fetch( + "openfe_analysis_simulation_output.tar.gz", + processor=pooch.Untar(), + ) + zenodo.fetch( + "openfe_analysis_skipped.tar.gz", + processor=pooch.Untar(), + ) + EOF + - name: "Run tests" run: | pytest -n auto -v --cov=openfe_analysis --cov-report=xml --durations=10 diff --git a/src/openfe_analysis/tests/conftest.py b/src/openfe_analysis/tests/conftest.py index 1a93ad7..f7a5cba 100644 --- a/src/openfe_analysis/tests/conftest.py +++ b/src/openfe_analysis/tests/conftest.py @@ -3,12 +3,9 @@ import pathlib import pooch import pytest -from filelock import FileLock POOCH_CACHE = pooch.os_cache("openfe_analysis") POOCH_CACHE.mkdir(parents=True, exist_ok=True) -LOCKFILE = POOCH_CACHE / "prepare.lock" -READY_FLAG = POOCH_CACHE / "data_ready.flag" ZENODO_RBFE_DATA = pooch.create( path=POOCH_CACHE, @@ -19,45 +16,24 @@ }, ) -def _prepare_data(): - """Download and extract large test data once per machine.""" - if READY_FLAG.exists(): - return +def _fetch_and_untar_once(filename: str) -> pathlib.Path: + untar_dir = POOCH_CACHE / f"{filename}.untar" - with FileLock(str(LOCKFILE)): - if READY_FLAG.exists(): - return + if not untar_dir.exists(): + ZENODO_RBFE_DATA.fetch(filename, processor=pooch.Untar()) - ZENODO_RBFE_DATA.fetch( - "openfe_analysis_simulation_output.tar.gz", - processor=pooch.Untar(), - ) - ZENODO_RBFE_DATA.fetch( - "openfe_analysis_skipped.tar.gz", - processor=pooch.Untar(), - ) - - READY_FLAG.touch() - - -_prepare_data() + return untar_dir @pytest.fixture(scope="session") def rbfe_output_data_dir() -> pathlib.Path: - return ( - POOCH_CACHE - / "openfe_analysis_simulation_output.tar.gz.untar" - / "openfe_analysis_simulation_output" - ) + untar_dir = _fetch_and_untar_once("openfe_analysis_simulation_output.tar.gz") + return untar_dir / "openfe_analysis_simulation_output" @pytest.fixture(scope="session") def rbfe_skipped_data_dir() -> pathlib.Path: - return ( - POOCH_CACHE - / "openfe_analysis_skipped.tar.gz.untar" - / "openfe_analysis_skipped" - ) + untar_dir = _fetch_and_untar_once("openfe_analysis_skipped.tar.gz") + return untar_dir / "openfe_analysis_skipped" @pytest.fixture(scope="session") def simulation_nc(rbfe_output_data_dir) -> pathlib.Path: From ead79515bb5630a54805aa17d343ea1c6d346e43 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 13:58:16 +0100 Subject: [PATCH 05/15] Run downloads before the testing to have a single download for all the runners for the matrix --- .github/workflows/ci.yaml | 83 ++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5163be8..e2d0b84 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,6 +16,48 @@ defaults: shell: bash -leo pipefail {0} jobs: + prefetch: + name: "Pre-fetch Zenodo Data" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Cache Pooch data + uses: actions/cache@v4 + with: + path: | + # linux cache location + ~/.cache/openfe_analysis + # osx cache location + ~/Library/Caches/openfe_analysis + # When files are added or changed in a pooch registry + # bump this key to create a new cache, for example if + # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 + key: pooch-${{ matrix.os }}-v1 + + - name: Pre-fetch Zenodo test data + run: | + python - <<'EOF' + import pooch + from pathlib import Path + + cache = Path(pooch.os_cache("openfe_analysis")) + cache.mkdir(parents=True, exist_ok=True) + + zenodo = pooch.create( + path=cache, + base_url="doi:10.5281/zenodo.17916322", + registry={ + "openfe_analysis_simulation_output.tar.gz": "md5:09752f2c4e5b7744d8afdee66dbd1414", + "openfe_analysis_skipped.tar.gz": "md5:3840d044299caacc4ccd50e6b22c0880", + }, + ) + + zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar()) + zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) + EOF + + tests: runs-on: ${{ matrix.os}} name: "💻-${{matrix.os }} 🐍-${{ matrix.python-version }}" @@ -56,47 +98,6 @@ jobs: run: | python -Ic "import openfe_analysis; print(openfe_analysis.__version__)" - - name: Cache Pooch data - uses: actions/cache@v4 - with: - path: | - # linux cache location - ~/.cache/openfe_analysis - # osx cache location - ~/Library/Caches/openfe_analysis - # When files are added or changed in a pooch registry - # bump this key to create a new cache, for example if - # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 - key: pooch-${{ matrix.os }}-v1 - - - name: Pre-fetch Zenodo test data - run: | - python - <<'EOF' - import pooch - from pathlib import Path - - cache = Path(pooch.os_cache("openfe_analysis")) - cache.mkdir(parents=True, exist_ok=True) - - zenodo = pooch.create( - path=cache, - base_url="doi:10.5281/zenodo.17916322", - registry={ - "openfe_analysis_simulation_output.tar.gz": "md5:09752f2c4e5b7744d8afdee66dbd1414", - "openfe_analysis_skipped.tar.gz": "md5:3840d044299caacc4ccd50e6b22c0880", - }, - ) - - zenodo.fetch( - "openfe_analysis_simulation_output.tar.gz", - processor=pooch.Untar(), - ) - zenodo.fetch( - "openfe_analysis_skipped.tar.gz", - processor=pooch.Untar(), - ) - EOF - - name: "Run tests" run: | pytest -n auto -v --cov=openfe_analysis --cov-report=xml --durations=10 From f898a35c7474f6f5d9c5c9fec234215b9959d290 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 14:00:58 +0100 Subject: [PATCH 06/15] add import pooch --- .github/workflows/ci.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2d0b84..b02c201 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,6 +22,14 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install pooch + run: pip install pooch + - name: Cache Pooch data uses: actions/cache@v4 with: From c675a5c0597390479820e205e311c1d14e4aa80e Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 14:11:27 +0100 Subject: [PATCH 07/15] Test out more --- .github/workflows/ci.yaml | 40 ++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b02c201..1f3303f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,15 +22,8 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - - - name: Install pooch - run: pip install pooch - - name: Cache Pooch data + id: pooch-cache uses: actions/cache@v4 with: path: | @@ -43,9 +36,23 @@ jobs: # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 key: pooch-${{ matrix.os }}-v1 + - name: "Setup Micromamba" + uses: mamba-org/setup-micromamba@v2 + with: + environment-file: environment.yml + environment-name: openfe_analysis_env + cache-environment: true + cache-downloads: true + cache-environment-key: environment-${{ steps.date.outputs.date }} + cache-downloads-key: downloads-${{ steps.date.outputs.date }} + create-args: >- + python=${{ matrix.python-version }} + init-shell: bash + - name: Pre-fetch Zenodo test data + if: steps.pooch-cache.outputs.cache-hit != 'true' run: | - python - <<'EOF' + micromamba run -n openfe_analysis_env python - <<'EOF' import pooch from pathlib import Path @@ -69,6 +76,7 @@ jobs: tests: runs-on: ${{ matrix.os}} name: "💻-${{matrix.os }} 🐍-${{ matrix.python-version }}" + needs: prefetch strategy: fail-fast: false matrix: @@ -85,6 +93,20 @@ jobs: id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" + + - name: Restore Pooch Cache + uses: actions/cache@v4 + with: + path: | + # linux cache location + ~/.cache/openfe_analysis + # osx cache location + ~/Library/Caches/openfe_analysis + # When files are added or changed in a pooch registry + # bump this key to create a new cache, for example if + # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 + key: pooch-${{ matrix.os }}-v1 + - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 with: From 129136161558893b5ce9b3832d63881918bc10f6 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:20:52 +0100 Subject: [PATCH 08/15] Test out something --- .github/workflows/ci.yaml | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f3303f..9c68496 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,20 +22,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Cache Pooch data - id: pooch-cache - uses: actions/cache@v4 - with: - path: | - # linux cache location - ~/.cache/openfe_analysis - # osx cache location - ~/Library/Caches/openfe_analysis - # When files are added or changed in a pooch registry - # bump this key to create a new cache, for example if - # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 - key: pooch-${{ matrix.os }}-v1 - - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 with: @@ -56,7 +42,7 @@ jobs: import pooch from pathlib import Path - cache = Path(pooch.os_cache("openfe_analysis")) + cache = Path.home() / ".cache/openfe_analysis" cache.mkdir(parents=True, exist_ok=True) zenodo = pooch.create( @@ -72,6 +58,12 @@ jobs: zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) EOF + - name: Upload Zenodo data as artifact + uses: actions/upload-artifact@v3 + with: + name: zenodo-data + path: ~/.cache/openfe_analysis + tests: runs-on: ${{ matrix.os}} @@ -93,19 +85,11 @@ jobs: id: date run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - - name: Restore Pooch Cache - uses: actions/cache@v4 + - name: Download Zenodo data artifact + uses: actions/download-artifact@v3 with: - path: | - # linux cache location - ~/.cache/openfe_analysis - # osx cache location - ~/Library/Caches/openfe_analysis - # When files are added or changed in a pooch registry - # bump this key to create a new cache, for example if - # the key is pooch-${{ matrix.os }}-1 change it to pooch-${{ matrix.os }}-2 - key: pooch-${{ matrix.os }}-v1 + name: zenodo-data + path: ~/.cache/openfe_analysis - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 From cbafa4bece2cf6bc79be0073267cf7f94f0db74c Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:22:02 +0100 Subject: [PATCH 09/15] Small fix --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c68496..0b4cbae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,7 +59,7 @@ jobs: EOF - name: Upload Zenodo data as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: zenodo-data path: ~/.cache/openfe_analysis @@ -86,7 +86,7 @@ jobs: run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - name: Download Zenodo data artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: zenodo-data path: ~/.cache/openfe_analysis From 1af61540377999986d36e4cc0f92a51639443c2c Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:47:45 +0100 Subject: [PATCH 10/15] Change os handling --- .github/workflows/ci.yaml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0b4cbae..c47c842 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,6 +11,10 @@ on: - cron: "0 7 * * *" workflow_dispatch: +# Normalize cache path for pooch across all OSes +env: + POOCH_CACHE_DIR: $HOME/.pooch + defaults: run: shell: bash -leo pipefail {0} @@ -38,11 +42,12 @@ jobs: - name: Pre-fetch Zenodo test data if: steps.pooch-cache.outputs.cache-hit != 'true' run: | + mkdir -p "$POOCH_CACHE_DIR/openfe_analysis" micromamba run -n openfe_analysis_env python - <<'EOF' import pooch from pathlib import Path - cache = Path.home() / ".cache/openfe_analysis" + cache = Path(pooch.os_cache("openfe_analysis")) cache.mkdir(parents=True, exist_ok=True) zenodo = pooch.create( @@ -62,7 +67,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: zenodo-data - path: ~/.cache/openfe_analysis + path: | + $POOCH_CACHE_DIR/openfe_analysis/openfe_analysis_simulation_output + $POOCH_CACHE_DIR/openfe_analysis/openfe_analysis_skipped tests: @@ -89,7 +96,7 @@ jobs: uses: actions/download-artifact@v4 with: name: zenodo-data - path: ~/.cache/openfe_analysis + path: $POOCH_CACHE_DIR/openfe_analysis - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 From 767212cae1aee98f15c5a1d30b6cbcb4f4ec6336 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:50:42 +0100 Subject: [PATCH 11/15] Add retry --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c47c842..2d82209 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,8 +59,8 @@ jobs: }, ) - zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar()) - zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) + zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar(), retry_if_failed=True) + zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar(), retry_if_failed=True) EOF - name: Upload Zenodo data as artifact From 3f9273f0f1d2d51226d3e3b40ef6e4f4fa9c6d57 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:52:22 +0100 Subject: [PATCH 12/15] Small fix --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d82209..4c705b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,8 +59,8 @@ jobs: }, ) - zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar(), retry_if_failed=True) - zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar(), retry_if_failed=True) + zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar(), timeout=30) + zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar(), timeout=30) EOF - name: Upload Zenodo data as artifact From b8a9cac3ae068f82c5da2101671e7fafae6880d5 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 15:55:30 +0100 Subject: [PATCH 13/15] Add retry --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c705b6..67555c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,10 +57,11 @@ jobs: "openfe_analysis_simulation_output.tar.gz": "md5:09752f2c4e5b7744d8afdee66dbd1414", "openfe_analysis_skipped.tar.gz": "md5:3840d044299caacc4ccd50e6b22c0880", }, + retry_if_failed = True, ) - zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar(), timeout=30) - zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar(), timeout=30) + zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar()) + zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) EOF - name: Upload Zenodo data as artifact From 9a7364afe7a248060566e9869af4300628d0fe61 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 16:17:44 +0100 Subject: [PATCH 14/15] Some more fixes --- .github/workflows/ci.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 67555c2..9b9adad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,6 +26,10 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Get current date + id: date + run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" + - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 with: @@ -42,7 +46,7 @@ jobs: - name: Pre-fetch Zenodo test data if: steps.pooch-cache.outputs.cache-hit != 'true' run: | - mkdir -p "$POOCH_CACHE_DIR/openfe_analysis" + mkdir -p "${{ env.POOCH_CACHE_DIR }}/openfe_analysis" micromamba run -n openfe_analysis_env python - <<'EOF' import pooch from pathlib import Path @@ -57,7 +61,6 @@ jobs: "openfe_analysis_simulation_output.tar.gz": "md5:09752f2c4e5b7744d8afdee66dbd1414", "openfe_analysis_skipped.tar.gz": "md5:3840d044299caacc4ccd50e6b22c0880", }, - retry_if_failed = True, ) zenodo.fetch("openfe_analysis_simulation_output.tar.gz", processor=pooch.Untar()) @@ -69,8 +72,8 @@ jobs: with: name: zenodo-data path: | - $POOCH_CACHE_DIR/openfe_analysis/openfe_analysis_simulation_output - $POOCH_CACHE_DIR/openfe_analysis/openfe_analysis_skipped + ${{ env.POOCH_CACHE_DIR }}/openfe_analysis/openfe_analysis_simulation_output + ${{ env.POOCH_CACHE_DIR }}/openfe_analysis/openfe_analysis_skipped tests: @@ -97,7 +100,7 @@ jobs: uses: actions/download-artifact@v4 with: name: zenodo-data - path: $POOCH_CACHE_DIR/openfe_analysis + path: ${{ env.POOCH_CACHE_DIR }}/openfe_analysis - name: "Setup Micromamba" uses: mamba-org/setup-micromamba@v2 From 5ff5824365d46b0ca2db7a2d6423b989cedf5468 Mon Sep 17 00:00:00 2001 From: hannahbaumann Date: Mon, 19 Jan 2026 18:19:10 +0100 Subject: [PATCH 15/15] Update ci yaml --- .github/workflows/ci.yaml | 75 ++++++++------------------------------- 1 file changed, 15 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9b9adad..8287532 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,20 +11,25 @@ on: - cron: "0 7 * * *" workflow_dispatch: -# Normalize cache path for pooch across all OSes -env: - POOCH_CACHE_DIR: $HOME/.pooch - defaults: run: shell: bash -leo pipefail {0} jobs: - prefetch: - name: "Pre-fetch Zenodo Data" - runs-on: ubuntu-latest + tests: + runs-on: ${{ matrix.os}} + name: "💻-${{matrix.os }} 🐍-${{ matrix.python-version }}" + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest"] + python-version: + - "3.11" + - "3.12" steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Get current date id: date @@ -43,11 +48,9 @@ jobs: python=${{ matrix.python-version }} init-shell: bash - - name: Pre-fetch Zenodo test data - if: steps.pooch-cache.outputs.cache-hit != 'true' + - name: "Download Zenodo data" run: | - mkdir -p "${{ env.POOCH_CACHE_DIR }}/openfe_analysis" - micromamba run -n openfe_analysis_env python - <<'EOF' + python - <<'EOF' import pooch from pathlib import Path @@ -67,54 +70,6 @@ jobs: zenodo.fetch("openfe_analysis_skipped.tar.gz", processor=pooch.Untar()) EOF - - name: Upload Zenodo data as artifact - uses: actions/upload-artifact@v4 - with: - name: zenodo-data - path: | - ${{ env.POOCH_CACHE_DIR }}/openfe_analysis/openfe_analysis_simulation_output - ${{ env.POOCH_CACHE_DIR }}/openfe_analysis/openfe_analysis_skipped - - - tests: - runs-on: ${{ matrix.os}} - name: "💻-${{matrix.os }} 🐍-${{ matrix.python-version }}" - needs: prefetch - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest", "macos-latest"] - python-version: - - "3.11" - - "3.12" - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Get current date - id: date - run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - - name: Download Zenodo data artifact - uses: actions/download-artifact@v4 - with: - name: zenodo-data - path: ${{ env.POOCH_CACHE_DIR }}/openfe_analysis - - - name: "Setup Micromamba" - uses: mamba-org/setup-micromamba@v2 - with: - environment-file: environment.yml - environment-name: openfe_analysis_env - cache-environment: true - cache-downloads: true - cache-environment-key: environment-${{ steps.date.outputs.date }} - cache-downloads-key: downloads-${{ steps.date.outputs.date }} - create-args: >- - python=${{ matrix.python-version }} - init-shell: bash - - name: "Install" run: | python -m pip install --no-deps . @@ -135,4 +90,4 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} file: coverage.xml fail_ci_if_error: false - verbose: true + verbose: true \ No newline at end of file