diff --git a/examples/input_convertors.ipynb b/examples/input_convertors.ipynb index b7eef8d5..85ea3b10 100644 --- a/examples/input_convertors.ipynb +++ b/examples/input_convertors.ipynb @@ -2,14 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import valenspy as vp \n", "from valenspy.input.converter import INPUT_CONVERTORS\n", "\n", - "from pathlib import Path" + "from pathlib import Path\n", + "import xarray as xr" ] }, { @@ -23,22 +24,1160 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "dict_keys(['ERA5', 'ERA5-Land', 'EOBS', 'CLIMATE_GRID', 'CCLM', 'ALARO_K'])" + "{'ERA5': ,\n", + " 'ERA5-Land': ,\n", + " 'EOBS': ,\n", + " 'CLIMATE_GRID': ,\n", + " 'CCLM': ,\n", + " 'ALARO_K': ,\n", + " 'RADCLIM': }" ] }, - "execution_count": 7, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "INPUT_CONVERTORS.keys() #These are the currently available input convertors" + "INPUT_CONVERTORS #These are the currently available input convertors" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 98MB\n",
+       "Dimensions:        (time: 744, bnds: 2, lon: 181, lat: 181)\n",
+       "Coordinates:\n",
+       "  * time           (time) datetime64[ns] 6kB 2014-01-01T00:15:00 ... 2014-01-...\n",
+       "  * lon            (lon) float32 724B -68.95 -68.85 -68.75 ... -51.05 -50.95\n",
+       "  * lat            (lat) float32 724B -12.05 -11.95 -11.85 ... 5.75 5.85 5.95\n",
+       "Dimensions without coordinates: bnds\n",
+       "Data variables:\n",
+       "    time_bnds      (time, bnds) datetime64[ns] 12kB ...\n",
+       "    lon_bnds       (lon, bnds) float32 1kB ...\n",
+       "    lat_bnds       (lat, bnds) float32 1kB ...\n",
+       "    precipitation  (time, lon, lat) float32 97MB ...\n",
+       "Attributes:\n",
+       "    CDI:                             Climate Data Interface version 2.2.4 (ht...\n",
+       "    Conventions:                     CF-1.6\n",
+       "    FileHeader:                      DOI=10.5067/GPM/IMERG/3B-HH/07;\\nDOIauth...\n",
+       "    FileInfo:                        DataFormatVersion=7e;\\nTKCodeBuildVersio...\n",
+       "    Grid.GridHeader:                 BinMethod=ARITHMETIC_MEAN;\\nRegistration...\n",
+       "    Grid.fullnamepath:               /Grid\n",
+       "    DODS_EXTRA.Unlimited_Dimension:  time\n",
+       "    history:                         Fri Oct 04 11:43:50 2024: cdo splitmon G...\n",
+       "    CDO:                             Climate Data Operators version 2.2.2 (ht...
" + ], + "text/plain": [ + " Size: 98MB\n", + "Dimensions: (time: 744, bnds: 2, lon: 181, lat: 181)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 6kB 2014-01-01T00:15:00 ... 2014-01-...\n", + " * lon (lon) float32 724B -68.95 -68.85 -68.75 ... -51.05 -50.95\n", + " * lat (lat) float32 724B -12.05 -11.95 -11.85 ... 5.75 5.85 5.95\n", + "Dimensions without coordinates: bnds\n", + "Data variables:\n", + " time_bnds (time, bnds) datetime64[ns] 12kB ...\n", + " lon_bnds (lon, bnds) float32 1kB ...\n", + " lat_bnds (lat, bnds) float32 1kB ...\n", + " precipitation (time, lon, lat) float32 97MB ...\n", + "Attributes:\n", + " CDI: Climate Data Interface version 2.2.4 (ht...\n", + " Conventions: CF-1.6\n", + " FileHeader: DOI=10.5067/GPM/IMERG/3B-HH/07;\\nDOIauth...\n", + " FileInfo: DataFormatVersion=7e;\\nTKCodeBuildVersio...\n", + " Grid.GridHeader: BinMethod=ARITHMETIC_MEAN;\\nRegistration...\n", + " Grid.fullnamepath: /Grid\n", + " DODS_EXTRA.Unlimited_Dimension: time\n", + " history: Fri Oct 04 11:43:50 2024: cdo splitmon G...\n", + " CDO: Climate Data Operators version 2.2.2 (ht..." + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = xr.open_dataset(Path(\"/data/gent/vo/002/gvo00202/master/data/Amazon/observations/satellite/GPM_2014_01.nc\"))\n", + "ds" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 98MB\n",
+       "Dimensions:        (time: 744, bnds: 2, lon: 181, lat: 181)\n",
+       "Coordinates:\n",
+       "  * time           (time) datetime64[ns] 6kB 2014-01-01T00:15:00 ... 2014-01-...\n",
+       "  * lon            (lon) float32 724B -68.95 -68.85 -68.75 ... -51.05 -50.95\n",
+       "  * lat            (lat) float32 724B -12.05 -11.95 -11.85 ... 5.75 5.85 5.95\n",
+       "Dimensions without coordinates: bnds\n",
+       "Data variables:\n",
+       "    time_bnds      (time, bnds) datetime64[ns] 12kB ...\n",
+       "    lon_bnds       (lon, bnds) float32 1kB ...\n",
+       "    lat_bnds       (lat, bnds) float32 1kB ...\n",
+       "    precipitation  (time, lon, lat) float32 97MB ...\n",
+       "Attributes:\n",
+       "    CDI:                             Climate Data Interface version 2.2.4 (ht...\n",
+       "    Conventions:                     CF-1.6\n",
+       "    FileHeader:                      DOI=10.5067/GPM/IMERG/3B-HH/07;\\nDOIauth...\n",
+       "    FileInfo:                        DataFormatVersion=7e;\\nTKCodeBuildVersio...\n",
+       "    Grid.GridHeader:                 BinMethod=ARITHMETIC_MEAN;\\nRegistration...\n",
+       "    Grid.fullnamepath:               /Grid\n",
+       "    DODS_EXTRA.Unlimited_Dimension:  time\n",
+       "    history:                         Fri Oct 04 11:43:50 2024: cdo splitmon G...\n",
+       "    CDO:                             Climate Data Operators version 2.2.2 (ht...
" + ], + "text/plain": [ + " Size: 98MB\n", + "Dimensions: (time: 744, bnds: 2, lon: 181, lat: 181)\n", + "Coordinates:\n", + " * time (time) datetime64[ns] 6kB 2014-01-01T00:15:00 ... 2014-01-...\n", + " * lon (lon) float32 724B -68.95 -68.85 -68.75 ... -51.05 -50.95\n", + " * lat (lat) float32 724B -12.05 -11.95 -11.85 ... 5.75 5.85 5.95\n", + "Dimensions without coordinates: bnds\n", + "Data variables:\n", + " time_bnds (time, bnds) datetime64[ns] 12kB ...\n", + " lon_bnds (lon, bnds) float32 1kB ...\n", + " lat_bnds (lat, bnds) float32 1kB ...\n", + " precipitation (time, lon, lat) float32 97MB ...\n", + "Attributes:\n", + " CDI: Climate Data Interface version 2.2.4 (ht...\n", + " Conventions: CF-1.6\n", + " FileHeader: DOI=10.5067/GPM/IMERG/3B-HH/07;\\nDOIauth...\n", + " FileInfo: DataFormatVersion=7e;\\nTKCodeBuildVersio...\n", + " Grid.GridHeader: BinMethod=ARITHMETIC_MEAN;\\nRegistration...\n", + " Grid.fullnamepath: /Grid\n", + " DODS_EXTRA.Unlimited_Dimension: time\n", + " history: Fri Oct 04 11:43:50 2024: cdo splitmon G...\n", + " CDO: Climate Data Operators version 2.2.2 (ht..." + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds = xr.open_dataset(Path(\"/data/gent/vo/002/gvo00202/master/data/Amazon/observations/satellite/GPM_2014_01.nc\"))\n", + "ds" ] }, { @@ -767,7 +1906,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "valenspy_dev", "language": "python", "name": "python3" }, @@ -781,7 +1920,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.10.0" } }, "nbformat": 4, diff --git a/src/valenspy/ancilliary_data/GPM_IMERG_lookup.yml b/src/valenspy/ancilliary_data/GPM_IMERG_lookup.yml new file mode 100644 index 00000000..b9cf5ab1 --- /dev/null +++ b/src/valenspy/ancilliary_data/GPM_IMERG_lookup.yml @@ -0,0 +1,15 @@ +# GPM_IMERG variable name conversion table based on CORDEX variables + +# Format + +# CORDEX VariableName: +# raw_name: +# raw_long_name: +# raw_units: + +# More info on GPM_IMERG: https://gpm.nasa.gov/data/imerg + +pr: + raw_name: precipitation + raw_long_name: Complete merged microwave-infrared (gauge-adjusted) precipitation estimate; formerly precipitationCal + raw_units: mm/hr \ No newline at end of file diff --git a/src/valenspy/input/converter.py b/src/valenspy/input/converter.py index de24071c..81567302 100644 --- a/src/valenspy/input/converter.py +++ b/src/valenspy/input/converter.py @@ -10,6 +10,7 @@ CCLM_to_CF, ALARO_K_to_CF, RADCLIM_to_CF, + GPM_IMERG_to_CF, ) @@ -52,6 +53,7 @@ def convert_input(self, data_sources, metadata_info=None): "CCLM": InputConverter(CCLM_to_CF), "ALARO_K": InputConverter(ALARO_K_to_CF), "RADCLIM": InputConverter(RADCLIM_to_CF), + "GPM_IMERG": InputConverter(GPM_IMERG_to_CF), } # Idea is to extend the shared functionality here (with subclasses if required) while the inputconvertor_functions are model specific. diff --git a/src/valenspy/input/converter_functions.py b/src/valenspy/input/converter_functions.py index 48452043..bdd55648 100644 --- a/src/valenspy/input/converter_functions.py +++ b/src/valenspy/input/converter_functions.py @@ -358,3 +358,32 @@ def RADCLIM_to_CF(ds: xr.Dataset, metadata_info=None) -> xr.Dataset: cf_status(ds) return ds + +def GPM_IMERG_to_CF(ds: xr.Dataset, metadata_info=None) -> xr.Dataset: + """ + Convert the GPM IMERG xarray netCDF to a CF compliant xarray Dataset + + Parameters + ---------- + ds : xr.Dataset + The xarray Dataset of GPM IMERG observations to convert + metadata_info : dict, optional + A dictionary containing additional metadata information to add to the dataset + + Returns + ------- + Dataset + The CF compliant GPM IMERG observations for the specified variable. + """ + obsdata_name = "GPM_IMERG" + raw_LOOKUP = load_yml(f"{obsdata_name}_lookup") + + if metadata_info is None: # Set standard metadata if not provided + metadata_info = {"freq": "hourly"} + + ds = convert_all_units_to_CF(ds, raw_LOOKUP, metadata_info) + ds = _set_global_attributes(ds, metadata_info) + + cf_status(ds) + + return ds \ No newline at end of file diff --git a/tests/push_tests/test_basic_unit_conversion_logic.py b/tests/push_tests/test_basic_unit_conversion_logic.py index 7c7e9907..99c66fac 100644 --- a/tests/push_tests/test_basic_unit_conversion_logic.py +++ b/tests/push_tests/test_basic_unit_conversion_logic.py @@ -3,7 +3,7 @@ import pytest -lookup_tables = ["EOBS_lookup", "CCLM_lookup", "ALARO-SFX_K_lookup","ERA5_lookup","CLIMATE_GRID_lookup", "RADCLIM_lookup"] +lookup_tables = ["EOBS_lookup", "CCLM_lookup", "ALARO-SFX_K_lookup","ERA5_lookup","CLIMATE_GRID_lookup", "RADCLIM_lookup", "GPM_IMERG_lookup"] @pytest.mark.parametrize("lookup_name", lookup_tables) def test_lookup_unit_conversion_coverage(lookup_name):