diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d462d751e..905350dd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: uses: astral-sh/setup-uv@v5 with: # Install a specific version of uv. - version: "0.7.13" + version: "0.11.8" - name: Run ruff (black) # Do not attempt to install the default dependencies, this is much faster. @@ -46,7 +46,7 @@ jobs: uses: astral-sh/setup-uv@v5 with: # Install a specific version of uv. - version: "0.7.13" + version: "0.11.8" - name: Check PR is linked to an issue # Send the PR number to the script, which will check if it is linked to an issue. @@ -62,7 +62,7 @@ jobs: uses: astral-sh/setup-uv@v5 with: # Install a specific version of uv. - version: "0.7.13" + version: "0.11.8" - name: Run all unit tests # Send the PR number to the script, which will check if it is linked to an issue. diff --git a/ci/cscs.yaml b/ci/cscs.yaml index eb2d444b4..302e18bb6 100644 --- a/ci/cscs.yaml +++ b/ci/cscs.yaml @@ -15,6 +15,7 @@ test_job: echo "PRIVATE_REPO_BRANCH=$PRIVATE_REPO_BRANCH" export WEATHERGEN_PRIVATE_REPO_PATH="$PWD/WeatherGenerator-private" echo "WEATHERGEN_PRIVATE_REPO_PATH=$WEATHERGEN_PRIVATE_REPO_PATH" + echo "uv version: $(uv --version) path: $(which uv)" git clone \ --depth 1 --single-branch \ @@ -22,9 +23,13 @@ test_job: https://oauth2:${PRIVATE_REPO_TOKEN}@gitlab.jsc.fz-juelich.de/esde/WeatherGenerator-private.git echo "Sync" - ./scripts/actions.sh sync + ./scripts/actions.sh sync-safe echo "Create links and run tests" ./scripts/actions.sh create-links + echo "Activate the environment" + source .venv/bin/activate + echo "Python version: $(python3 --version) path: $(which python3)" + echo "Pytest version: $(pytest --version) path: $(which pytest)" echo "Run tests" ./scripts/actions.sh integration-test${STAGE_TYPE:-} variables: diff --git a/packages/common/pyproject.toml b/packages/common/pyproject.toml index 3bc52c764..58a5ad407 100644 --- a/packages/common/pyproject.toml +++ b/packages/common/pyproject.toml @@ -8,7 +8,8 @@ dependencies = [ "xarray>=2025.6.1", "dask>=2024.9.1", "zarr~=3.1.3", - "numcodecs<0.16.0", + # zarr3 has issues with more recent numcodecs + "numcodecs<=0.16.5", "astropy_healpix~=1.1.2", "omegaconf~=2.3.0", "pyyaml", diff --git a/pyproject.toml b/pyproject.toml index 00103cb8c..45c553f65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ dependencies = [ 'tqdm', 'matplotlib', 'packaging', - 'wheel', 'psutil', "polars~=1.25.2", "omegaconf~=2.3.0", @@ -27,7 +26,13 @@ dependencies = [ "anemoi-datasets", "weathergen-common", "weathergen-evaluate", - "weathergen-readers-extra" + "weathergen-readers-extra", + "pytest>=9.0.2", + # older version have no precompiled linux-aarch64 cp312 wheel + "numcodecs>=0.16", + # This seems necessary for build isolation? + 'wheel', + 'setuptools', ] @@ -52,6 +57,7 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/weathergen"] + [dependency-groups] # The development dependencies dev = [ @@ -161,18 +167,19 @@ enable = ["W0201", "W0141"] [tool.uv] # Most work is done a distributed filesystem, where hardlink is not always possible. # Also, trying to resolve some permissions issue, see 44. +# symlink: faster than copy (ex: torch), but leads to files being deleted +# when the cache is in SCRATCH with a deletion policy. link-mode = "symlink" # This guarantees that the build is deterministic and will not be impacted # by future releases of dependencies or sub-dependencies. # See https://docs.astral.sh/uv/reference/settings/#exclude-newer -# TODO: pytorch does not publish valid release timestamps, so sadly it does not work. exclude-newer = "2026-02-27T00:00:00Z" # The minimum version of uv required. # It is tightly controlled because the format of uv.lock has changed # over revisions, causing reformats to happen without reason. # Also, relatively recent versions are required to support workspaces. -required-version = ">=0.7.0" +required-version = ">=0.11.0" # The supported environments # TODO: add macos and windows (CPU only, for running tests) @@ -191,12 +198,22 @@ conflicts = [ ] +# These packages don't declare setuptools/wheel in build-system.requires, so +# isolated builds fail with "No module named 'setuptools.build_meta'". +# setuptools is pinned <76: with newer setuptools (>=82) the +# `setuptools.build_meta:__legacy__` backend has been removed +[tool.uv.extra-build-dependencies] +anemoi-datasets = ["setuptools<76", "wheel"] +cfunits = ["setuptools<76", "wheel"] +antlr4-python3-runtime = ["setuptools<76", "wheel"] +# Cartopy requires setuptools >= 77 +cartopy = ["setuptools<78", "wheel"] + [[tool.uv.index]] name = "pytorch-cu126" url = "https://download.pytorch.org/whl/cu126" explicit = true - [tool.pyrefly] project-includes = ["src/"] project-excludes = [ @@ -254,9 +271,11 @@ torch = [ # TODO: explore pytorch + metal backend { index = "pytorch-cpu", marker = "sys_platform == 'darwin'", extra="gpu"}, ] -# branch = "feature/any-zarr-version-in-pyproject/" +# TODO: switch back to this branch, but this is a larger update and the tag name is incorrect +# anemoi-datasets = { git = "https://github.com/ecmwf/anemoi-datasets", tag = "v0.0.2-special-zarr3"} anemoi-datasets = { git = "https://github.com/tjhunter/anemoi-datasets", tag = "0.0.1"} + [tool.pytest.ini_options] log_cli = true log_cli_level = "INFO" diff --git a/scripts/actions.sh b/scripts/actions.sh index 29fd22a72..abea30ca1 100755 --- a/scripts/actions.sh +++ b/scripts/actions.sh @@ -8,6 +8,7 @@ case "$1" in sync) ( cd "$SCRIPT_DIR" || exit 1 + # Creates a virtual environment without checking the integrity of the cache. # If we are running on a mac, use the cpu extra if [[ "$(uname)" == "Darwin" ]]; then uv sync --all-packages --extra cpu @@ -17,6 +18,28 @@ case "$1" in uv sync --all-packages --extra gpu ) ;; + sync-safe) + ( + # Creates a virtual environment, checking the integrity of the cache + # and copying the files. + # This is slower (+ 60 seconds to the sync) but it is prevents issues with + # corrupted cache. These issues happen when a shared filesystem such as LUSTRE + # is used along with symlinks and a SCRATCH deletion policy: some files from + # cached packages may get deleted because they seem to not be touched enough. + cd "$SCRIPT_DIR" || exit 1 + # --refresh --reinstall : LUSTRE may clean up some pieces of the cache. + # This ensures basic integrity but it is too slow (adds 60 seconds to the sync) to do it on every sync. So we only do it on mac, where the cache is more likely to get corrupted. + # --link-mode=copy overrides the pyproject.toml setting (symlink) to fully + # detach installed files from the cache, so SCRATCH cleanups can't corrupt the venv. + # If we are running on a mac, use the cpu extra + if [[ "$(uname)" == "Darwin" ]]; then + uv sync --all-packages --extra cpu --refresh --reinstall --link-mode=copy + exit 0 + fi + # Otherwise, use the gpu extra + uv sync --all-packages --extra gpu --refresh --reinstall --link-mode=copy + ) + ;; lint) ( cd "$SCRIPT_DIR" || exit 1 @@ -128,7 +151,11 @@ case "$1" in # 1. Get the path of the private config of the cluster # 2. Read the yaml and extract the path of the shared conf # This uses the yq command. It is a python package so uvx (bundled with uv) will donwload and create the right venv - export working_dir=$(cat $("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config) | uvx yq .path_shared_working_dir) + # The 'yq' command is used in a separate virtual environment, because the cache + # around that tool can get corrupted. + # See https://github.com/ecmwf/WeatherGenerator/issues/2298 + export working_dir=$(cat "$("$PRIVATE_REPO_PATH"/hpc/platform-env.py hpc-config)" | + UV_CACHE_DIR="$(mktemp -d)" VIRTUAL_ENV="" uvx yq .path_shared_working_dir) # Remove quotes export working_dir=$(echo "$working_dir" | sed 's/[\"\x27]//g') # If the working directory does not exist, exit with an error @@ -177,7 +204,7 @@ case "$1" in *) ( # Automatically extract all options from the case statement - options=$(grep -oP '^\s*\K[\w-]+(?=\))' "$0" | tr '\n' '|' | sed 's/|$//') + options=$(sed -nE 's/^[[:space:]]*([a-zA-Z][a-zA-Z0-9_-]*)\).*/\1/p' "$0" | tr '\n' '|' | sed 's/|$//') echo "Usage: $0 {$options}" exit 1 )