From 83774c7c40ae116225b4ddf030687a204aa74d4d Mon Sep 17 00:00:00 2001 From: malmans2 Date: Fri, 5 Dec 2025 16:53:56 +0100 Subject: [PATCH 1/4] Migrate to ecmwf-datastores-client --- ci/combined-environment-ci.yml | 23 +++++++++++++++++++++++ environment.yml | 2 +- pyproject.toml | 3 +-- xarray_ecmwf/client_cdsapi.py | 8 ++++---- xarray_ecmwf/engine_ecmwf.py | 4 ++-- 5 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 ci/combined-environment-ci.yml diff --git a/ci/combined-environment-ci.yml b/ci/combined-environment-ci.yml new file mode 100644 index 0000000..241cf72 --- /dev/null +++ b/ci/combined-environment-ci.yml @@ -0,0 +1,23 @@ +channels: +- conda-forge +- nodefaults +dependencies: +- attrs +- cfgrib +- ecmwf-datastores-client +- ecmwf-opendata +- jsonschema +- make +- mypy +- myst-parser +- pandas-stubs +- pip +- pre-commit +- pydata-sphinx-theme +- pytest +- pytest-cov +- sphinx +- sphinx-autoapi +- xarray +- pip: + - polytope-client diff --git a/environment.yml b/environment.yml index 2662f34..0f71e12 100644 --- a/environment.yml +++ b/environment.yml @@ -9,8 +9,8 @@ channels: # DO NOT EDIT ABOVE THIS LINE, ADD DEPENDENCIES BELOW AS SHOWN IN THE EXAMPLE dependencies: - attrs -- cdsapi - cfgrib +- ecmwf-datastores-client - ecmwf-opendata - pip - xarray diff --git a/pyproject.toml b/pyproject.toml index 65bce00..3451a5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering" ] -dependencies = ["cdsapi", "cfgrib", "ecmwf-opendata", "polytope-client", "xarray"] +dependencies = ["cfgrib", "ecmwf-datastores-client", "ecmwf-opendata", "polytope-client", "xarray"] description = "Xarray backend to access data via the cdsapi package" dynamic = ["version"] license = {file = "LICENSE"} @@ -32,7 +32,6 @@ strict = true [[tool.mypy.overrides]] ignore_missing_imports = true module = [ - "cdsapi", "cf2cdm", "ecmwf", "ecmwf.opendata", diff --git a/xarray_ecmwf/client_cdsapi.py b/xarray_ecmwf/client_cdsapi.py index 0ce1cdd..202ac89 100644 --- a/xarray_ecmwf/client_cdsapi.py +++ b/xarray_ecmwf/client_cdsapi.py @@ -3,7 +3,7 @@ from typing import Any import attrs -import cdsapi +import ecmwf.datastores import numpy as np import xarray as xr @@ -16,13 +16,13 @@ @attrs.define class CdsapiRequestClient: - client_kwargs: dict[str, Any] = {"quiet": True, "retry_max": 1} + client_kwargs: dict[str, Any] = {"quiet": True, "maximum_tries": 1} def submit_and_wait_on_result(self, request: dict[str, Any]) -> Any: request = request.copy() dataset = request.pop("dataset") - client = cdsapi.Client(**self.client_kwargs) - return client.retrieve(dataset, request | {"format": "grib"}) + client = ecmwf.datastores.Client(**self.client_kwargs) + return client.submit_and_wait_on_results(dataset, request | {"format": "grib"}) def get_filename(self, result: Any) -> str: return result.location.split("/")[-1] # type: ignore diff --git a/xarray_ecmwf/engine_ecmwf.py b/xarray_ecmwf/engine_ecmwf.py index ef3e89a..a270abc 100644 --- a/xarray_ecmwf/engine_ecmwf.py +++ b/xarray_ecmwf/engine_ecmwf.py @@ -108,7 +108,7 @@ def retrieve_once( if not os.path.isdir(self.cache_folder): os.makedirs(self.cache_folder, exist_ok=True) - with xr.backends.locks.get_write_lock(f"{HOSTNAME}-grib"): # type: ignore + with xr.backends.locks.get_write_lock(f"{HOSTNAME}-grib"): if not os.path.exists(path): robust_save_to_file(self.request_client.download, (result,), path) ds = self.open_dataset(path) @@ -142,7 +142,7 @@ def cached_empty_dataset(self, request: dict[str, Any]) -> Iterator[xr.Dataset]: if not os.path.exists(path): with self.retrieve(request) as read_ds: # check again as the retrieve may be long - with xr.backends.locks.get_write_lock(f"{HOSTNAME}-zarr"): # type: ignore + with xr.backends.locks.get_write_lock(f"{HOSTNAME}-zarr"): if not os.path.exists(path): # NOTE: be sure that read_ds is chunked so compute=False only # writes the metadata. Some open_dataset From 082e76010e92718386ce45b3a7cc8d8dd0199dac Mon Sep 17 00:00:00 2001 From: malmans2 Date: Fri, 5 Dec 2025 16:56:29 +0100 Subject: [PATCH 2/4] remove combined env --- ci/combined-environment-ci.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 ci/combined-environment-ci.yml diff --git a/ci/combined-environment-ci.yml b/ci/combined-environment-ci.yml deleted file mode 100644 index 241cf72..0000000 --- a/ci/combined-environment-ci.yml +++ /dev/null @@ -1,23 +0,0 @@ -channels: -- conda-forge -- nodefaults -dependencies: -- attrs -- cfgrib -- ecmwf-datastores-client -- ecmwf-opendata -- jsonschema -- make -- mypy -- myst-parser -- pandas-stubs -- pip -- pre-commit -- pydata-sphinx-theme -- pytest -- pytest-cov -- sphinx -- sphinx-autoapi -- xarray -- pip: - - polytope-client From 5cf714c2150412049512e7a395df66d6f471033f Mon Sep 17 00:00:00 2001 From: malmans2 Date: Fri, 5 Dec 2025 16:57:41 +0100 Subject: [PATCH 3/4] remove quiet arg --- xarray_ecmwf/client_cdsapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray_ecmwf/client_cdsapi.py b/xarray_ecmwf/client_cdsapi.py index 202ac89..db56c4c 100644 --- a/xarray_ecmwf/client_cdsapi.py +++ b/xarray_ecmwf/client_cdsapi.py @@ -16,7 +16,7 @@ @attrs.define class CdsapiRequestClient: - client_kwargs: dict[str, Any] = {"quiet": True, "maximum_tries": 1} + client_kwargs: dict[str, Any] = {"maximum_tries": 1} def submit_and_wait_on_result(self, request: dict[str, Any]) -> Any: request = request.copy() From 2b944caa51e9cb2f496610e456701b178e5ddb86 Mon Sep 17 00:00:00 2001 From: malmans2 Date: Fri, 5 Dec 2025 17:08:44 +0100 Subject: [PATCH 4/4] fix docs --- README.md | 4 ++-- ci/environment-integration.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 928a117..2489a90 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ This Open Source project is build by B-Open - https://www.bopen.eu xarray-ecmwf is a Python library and [Xarray](https://docs.xarray.dev) backend with the following functionalities: - opens an ECMWF style request as a Xarray Dataset connected to the remote services - - the [Climate Data Store](https://cds.climate.copernicus.eu) via [cdsapi](https://github.com/ecmwf/cdsapi): ERA5, Seasonal forecasts - - the [Athospheric Data Store](https://ads.atmosphere.copernicus.eu) via cdsapi + - the [Climate Data Store](https://cds.climate.copernicus.eu) via [ecmwf-datastores-client](https://github.com/ecmwf/ecmwf-datastores-client): ERA5, Seasonal forecasts + - the [Atmospheric Data Store](https://ads.atmosphere.copernicus.eu) via ecmwf-datastores-client - the [ECMWF Open data](https://www.ecmwf.int/en/forecasts/datasets/open-data) via [ecmwf-opendata](https://github.com/ecmwf/ecmwf-opendata): High resolution forecasts, ensemble forecast - allows lazy loading the data and well integrated with [Dask](https://www.dask.org) and [Dask.distributed](https://distributed.dask.org) - allows chunking the input request according to a configurable splitting strategy. Allowed strategies: diff --git a/ci/environment-integration.yml b/ci/environment-integration.yml index 810ab03..fcfc5ca 100644 --- a/ci/environment-integration.yml +++ b/ci/environment-integration.yml @@ -7,8 +7,8 @@ dependencies: - pytest - pytest-cov # DO NOT EDIT ABOVE THIS LINE, ADD DEPENDENCIES BELOW -- cdsapi - ecmwf-api-client +- ecmwf-datastores-client - ecmwf-opendata - pandas-stubs - pip diff --git a/pyproject.toml b/pyproject.toml index 3451a5e..8ee2a62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ classifiers = [ "Topic :: Scientific/Engineering" ] dependencies = ["cfgrib", "ecmwf-datastores-client", "ecmwf-opendata", "polytope-client", "xarray"] -description = "Xarray backend to access data via the cdsapi package" +description = "Xarray backend to access ECMWF datastores" dynamic = ["version"] license = {file = "LICENSE"} name = "xarray-ecmwf"