[2298] Fix CI

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Run ruff (black)
         # Do not attempt to install the default dependencies, this is much faster.
@@ -46,7 +46,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Check PR is linked to an issue
       # Send the PR number to the script, which will check if it is linked to an issue.
@@ -62,7 +62,7 @@ jobs:
         uses: astral-sh/setup-uv@v5
         with:
           # Install a specific version of uv.
-          version: "0.7.13"
+          version: "0.11.8"
 
       - name: Run all unit tests
       # Send the PR number to the script, which will check if it is linked to an issue.

ci/cscs.yaml

@@ -15,16 +15,21 @@ test_job:
     echo "PRIVATE_REPO_BRANCH=$PRIVATE_REPO_BRANCH"
     export WEATHERGEN_PRIVATE_REPO_PATH="$PWD/WeatherGenerator-private"
     echo "WEATHERGEN_PRIVATE_REPO_PATH=$WEATHERGEN_PRIVATE_REPO_PATH"
+    echo "uv version: $(uv --version) path: $(which uv)"
 
     git clone \
       --depth 1 --single-branch \
       --branch "$PRIVATE_REPO_BRANCH" \
       https://oauth2:${PRIVATE_REPO_TOKEN}@gitlab.jsc.fz-juelich.de/esde/WeatherGenerator-private.git
 
     echo "Sync"
-    ./scripts/actions.sh sync
+    ./scripts/actions.sh sync-safe
     echo "Create links and run tests"
     ./scripts/actions.sh create-links
+    echo "Activate the environment"
+    source .venv/bin/activate
+    echo "Python version: $(python3 --version) path: $(which python3)"
+    echo "Pytest version: $(pytest --version) path: $(which pytest)"
     echo "Run tests"
     ./scripts/actions.sh integration-test${STAGE_TYPE:-}
   variables:

packages/common/pyproject.toml

@@ -8,7 +8,8 @@ dependencies = [
     "xarray>=2025.6.1",
     "dask>=2024.9.1",
     "zarr~=3.1.3",
-    "numcodecs<0.16.0",
+    # zarr3 has issues with more recent numcodecs
+    "numcodecs<=0.16.5",
     "astropy_healpix~=1.1.2",
     "omegaconf~=2.3.0",
     "pyyaml",

pyproject.toml

@@ -17,7 +17,6 @@ dependencies = [
  'tqdm',
  'matplotlib',
  'packaging',
- 'wheel',
  'psutil',
  "polars~=1.25.2",
  "omegaconf~=2.3.0",
@@ -27,7 +26,13 @@ dependencies = [
  "anemoi-datasets",
  "weathergen-common",
  "weathergen-evaluate",
- "weathergen-readers-extra"
+ "weathergen-readers-extra",
+ "pytest>=9.0.2",
+ # older version have no precompiled linux-aarch64 cp312 wheel 
+ "numcodecs>=0.16",
+ # This seems necessary for build isolation?
+ 'wheel',
+ 'setuptools',
 ]
 
 
@@ -52,6 +57,7 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = ["src/weathergen"]
 
+
 [dependency-groups]
 # The development dependencies
 dev = [
@@ -161,18 +167,19 @@ enable = ["W0201", "W0141"]
 [tool.uv]
 # Most work is done a distributed filesystem, where hardlink is not always possible.
 # Also, trying to resolve some permissions issue, see 44.
+# symlink: faster than copy (ex: torch), but leads to files being deleted
+# when the cache is in SCRATCH with a deletion policy.
 link-mode = "symlink"
 # This guarantees that the build is deterministic and will not be impacted 
 # by future releases of dependencies or sub-dependencies. 
 # See https://docs.astral.sh/uv/reference/settings/#exclude-newer
-# TODO: pytorch does not publish valid release timestamps, so sadly it does not work.
 exclude-newer = "2026-02-27T00:00:00Z"
 
 # The minimum version of uv required.
 # It is tightly controlled because the format of uv.lock has changed 
 # over revisions, causing reformats to happen without reason.
 # Also, relatively recent versions are required to support workspaces.
-required-version = ">=0.7.0"
+required-version = ">=0.11.0"
 
 # The supported environments
 # TODO: add macos and windows (CPU only, for running tests)
@@ -191,12 +198,22 @@ conflicts = [
 ]
 
 
+# These packages don't declare setuptools/wheel in build-system.requires, so
+# isolated builds fail with "No module named 'setuptools.build_meta'".
+# setuptools is pinned <76: with newer setuptools (>=82) the
+# `setuptools.build_meta:__legacy__` backend has been removed
+[tool.uv.extra-build-dependencies]
+anemoi-datasets = ["setuptools<76", "wheel"]
+cfunits = ["setuptools<76", "wheel"]
+antlr4-python3-runtime = ["setuptools<76", "wheel"]
+# Cartopy  requires setuptools >= 77
+cartopy = ["setuptools<78", "wheel"]
+
 [[tool.uv.index]]
 name = "pytorch-cu126"
 url = "https://download.pytorch.org/whl/cu126"
 explicit = true
 
-
 [tool.pyrefly]
 project-includes = ["src/"]
 project-excludes = [
@@ -254,9 +271,11 @@ torch = [
   # TODO: explore pytorch + metal backend
   { index = "pytorch-cpu", marker = "sys_platform == 'darwin'", extra="gpu"},
 ]
-# branch = "feature/any-zarr-version-in-pyproject/"
+# TODO: switch back to this branch, but this is a larger update and the tag name is incorrect
+# anemoi-datasets = { git = "https://github.com/ecmwf/anemoi-datasets", tag = "v0.0.2-special-zarr3"}
 anemoi-datasets = { git = "https://github.com/tjhunter/anemoi-datasets", tag = "0.0.1"}
 
+
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "INFO"

scripts/actions.sh

@@ -8,6 +8,7 @@ case "$1" in
   sync)
     (
       cd "$SCRIPT_DIR" || exit 1
+      # Creates a virtual environment without checking the integrity of the cache.
       # If we are running on a mac, use the cpu extra
       if [[ "$(uname)" == "Darwin" ]]; then
         uv sync --all-packages --extra cpu
@@ -17,6 +18,28 @@ case "$1" in
       uv sync --all-packages --extra gpu
     )
     ;;
+  sync-safe)
+    (
+      # Creates a virtual environment, checking the integrity of the cache 
+      # and copying the files.
+      # This is slower (+ 60 seconds to the sync) but it is prevents issues with 
+      # corrupted cache. These issues happen when a shared filesystem such as LUSTRE
+      # is used along with symlinks and a SCRATCH deletion policy: some files from 
+      # cached packages may get deleted because they seem to not be touched enough.
+      cd "$SCRIPT_DIR" || exit 1
+      # --refresh --reinstall : LUSTRE may clean up some pieces of the cache.
+      # This ensures basic integrity but it is too slow (adds 60 seconds to the sync) to do it on every sync. So we only do it on mac, where the cache is more likely to get corrupted.
+      # --link-mode=copy overrides the pyproject.toml setting (symlink) to fully
+      # detach installed files from the cache, so SCRATCH cleanups can't corrupt the venv.
+      # If we are running on a mac, use the cpu extra
+      if [[ "$(uname)" == "Darwin" ]]; then
+        uv sync --all-packages --extra cpu --refresh --reinstall --link-mode=copy
+        exit 0
+      fi
+      # Otherwise, use the gpu extra
+      uv sync --all-packages --extra gpu --refresh --reinstall --link-mode=copy
+    )
+    ;;
   lint)
     (
       cd "$SCRIPT_DIR" || exit 1
@@ -128,7 +151,11 @@ case "$1" in
       # 1. Get the path of the private config of the cluster
       # 2. Read the yaml and extract the path of the shared conf
       # This uses the yq command. It is a python package so uvx (bundled with uv) will donwload and create the right venv
-      export working_dir=$(cat $("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config) | uvx yq .path_shared_working_dir)
+      # The 'yq' command is used in a separate virtual environment, because the cache 
+      # around that tool can get corrupted. 
+      # See https://github.com/ecmwf/WeatherGenerator/issues/2298
+      export working_dir=$(cat "$("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config)" |
+        UV_CACHE_DIR="$(mktemp -d)" VIRTUAL_ENV=""  uvx yq .path_shared_working_dir)      
       # Remove quotes
       export working_dir=$(echo "$working_dir" | sed 's/[\"\x27]//g')
       # If the working directory does not exist, exit with an error
@@ -177,7 +204,7 @@ case "$1" in
   *)
     (
       # Automatically extract all options from the case statement
-      options=$(grep -oP '^\s*\K[\w-]+(?=\))' "$0" | tr '\n' '|' | sed 's/|$//')
+      options=$(sed -nE 's/^[[:space:]]*([a-zA-Z][a-zA-Z0-9_-]*)\).*/\1/p' "$0" | tr '\n' '|' | sed 's/|$//')
       echo "Usage: $0 {$options}"
       exit 1
     )