Add lapse-rate correction for temperature

.jretrievedwh-conf.prod.py

@@ -0,0 +1,48 @@
+import base64
+import json
+import os
+import urllib.request
+from pathlib import Path
+
+
+def _read_dotenv(path):
+    result = {}
+    try:
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                key, _, value = line.partition("=")
+                result[key.strip()] = value.strip().strip('"').strip("'")
+    except FileNotFoundError:
+        pass
+    return result
+
+
+_client_id = os.environ.get("JRETRIEVE_CLIENT_ID")
+_client_secret = os.environ.get("JRETRIEVE_CLIENT_SECRET")
+if not _client_id or not _client_secret:
+    _dotenv = _read_dotenv(Path(os.environ.get("JRETRIEVE_CONF_DIR", ".")) / ".env")
+    _client_id = _client_id or _dotenv.get("JRETRIEVE_CLIENT_ID")
+    _client_secret = _client_secret or _dotenv.get("JRETRIEVE_CLIENT_SECRET")
+if not _client_id or not _client_secret:
+    raise RuntimeError(
+        "jretrieve credentials not found. Set JRETRIEVE_CLIENT_ID and "
+        "JRETRIEVE_CLIENT_SECRET in the environment or in a .env file next "
+        "to this script."
+    )
+
+jretrieve_url = "https://service.meteoswiss.ch/jretrieve/api/v1"
+with urllib.request.urlopen(
+    urllib.request.Request(
+        method="POST",
+        url="https://service.meteoswiss.ch/auth/realms/meteoswiss.ch/protocol/openid-connect/token",
+        data=b"grant_type=client_credentials",
+        headers={
+            b"Authorization": b"Basic "
+            + base64.b64encode(f"{_client_id}:{_client_secret}".encode())
+        },
+    )
+) as f:
+    auth_header = "Bearer " + json.loads(f.read().decode())["access_token"]

README.md

@@ -145,6 +145,31 @@ You can then run it with:
 evalml experiment path/to/experiment/config.yaml --report
 ```
 
+### Truth sources
+
+The `truth.root` value selects how the ground truth is loaded:
+
+- **Analysis Zarr** — a path ending in `.zarr` (anemoi analysis dataset).
+- **DWH / jretrievedwh** — a `jretrievedwh:` marker string fetching surface observations
+  (e.g. SMN) live from the MeteoSwiss data warehouse. Variables are mapped to
+  ICON names in SI units (temperatures in K, pressure in Pa, precipitation as the hourly
+  sum); wind `U_10M`/`V_10M` are derived from speed + direction.
+
+  Marker syntax (station selection is required; pick one of group/locations/bbox):
+
+  ```yaml
+  truth:
+    label: SwissMetNet
+    root: jretrievedwh:1,2                          # stn_group_id (default)
+    # root: jretrievedwh:locations=ARO,KLO,LUG      # explicit nat_abbr list
+    # root: jretrievedwh:bbox=45.8,47.8,5.9,10.5    # minlat,maxlat,minlon,maxlon
+  ```
+
+  **Prerequisites:** `jretrievedwh.py` must be on `$PATH` (falls back to
+  `/oprusers/osm/opr.inn/bin/jretrievedwh.py`) and DWH credentials must be
+  available — see [Credentials setup](#credentials-setup). No data is
+  pre-downloaded — the obs are queried at verification time.
+
 
 ## Installation
 
@@ -175,6 +200,25 @@ valid for 30 days. Every training or evaluation run within this period automatic
 extends the token by another 30 days. It’s good practice to run the login command before
 executing the workflow to ensure your token is still valid.
 
+### DWH / jretrieve credentials
+
+To use `jretrievedwh:` as a truth source, provide a client ID and secret for
+the MeteoSwiss service account. Set them as environment variables:
+
+```bash
+export JRETRIEVE_CLIENT_ID=your-client-id
+export JRETRIEVE_CLIENT_SECRET=your-client-secret
+```
+
+or place them in a `.env` file at the project root (next to `.jretrievedwh-conf.prod.py`):
+
+```
+JRETRIEVE_CLIENT_ID=your-client-id
+JRETRIEVE_CLIENT_SECRET=your-client-secret
+```
+
+The `.env` file is listed in `.gitignore` and is never committed.
+
 ## Workspace setup
 
 By default, data produced by the workflow will be stored under `output/` in your working directory.

config/varda-single-1.0.yaml

@@ -37,7 +37,12 @@ runs:
 
 truth:
   label: SwissMetNet
-  root: output/data/observations/peakweather
+  root: jretrievedwh:1,2
+  # To verify against SwissMetNet observations from the DWH via jretrievedwh,
+  # set instead (requires jretrievedwh.py on $PATH and $OPR_HOME set):
+  # Other selectors: root: jretrievedwh:locations=ARO,KLO,LUG
+  #                  root: jretrievedwh:bbox=45.8,47.8,5.9,10.5
+  #                  append ;stage=devt to target a non-prod DWH stage
 
 experiment:
   params:

pyproject.toml

@@ -22,7 +22,6 @@ dependencies = [
     "pyproj>=3.7.2",
     "marimo>=0.23.3",
     "geopandas>=0.14.0",
-    "peakweather",
     "pyzmq>=27.1.0",
     "scores>=2.0.0",
     "eccodes>=2.44,<2.48",
@@ -60,5 +59,4 @@ packages = [
 ]
 
 [tool.uv.sources]
-peakweather = { git = "https://github.com/MeteoSwiss/PeakWeather.git" }
 eccodes-cosmo-resources-python = { git = "https://github.com/MeteoSwiss/eccodes-cosmo-resources-python" }