From bc4b9b00068da1e0cfbb0cdb1085bc0590822b6b Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Tue, 12 May 2026 14:49:58 -0400 Subject: [PATCH 1/7] update generate-charts endpoint for return periods --- climatedata_api/charts.py | 65 ++++++++++++++++++++++++++++++++++----- default_settings.py | 20 ++++++++++++ 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index 90b38f5..e3bc47b 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -44,7 +44,8 @@ def _format_slices_to_highcharts_series(observations_location_slice, bccaq_locat chart_series['modeled_historical_range'] = convert_time_series_dataset_to_list( xr.merge([bccaq_location_slice_historical[f'{scenarios[0]}_{var}_p10'], bccaq_location_slice_historical[f'{scenarios[0]}_{var}_p90']]), decimals) - # we return values in historical for all scenarios after HISTORICAL_DATE_LIMIT + + # For projection data, filter to only include values after the historical period (HISTORICAL_DATE_LIMIT) bccaq_location_slice = bccaq_location_slice.where( bccaq_location_slice.time >= np.datetime64(app.config['HISTORICAL_DATE_LIMIT_AFTER'][dataset_name]), drop=True) @@ -80,6 +81,47 @@ def _format_slices_to_highcharts_series(observations_location_slice, bccaq_locat return chart_series +def _format_slices_to_highcharts_series_return_periods(observations_location_slice, delta_30y_slice, var, decimals, dataset_name): + """ + Format observations, bccaq and delta_30y slices to dictionary of series ready for highcharts + """ + scenarios = app.config['SCENARIOS'][dataset_name] + delta_naming = app.config['DELTA_NAMING'][dataset_name] + + chart_series = {} + + if observations_location_slice[var].attrs.get('units') == 'K': + observations_location_slice = observations_location_slice + app.config['KELVIN_TO_C'] + + chart_series['30y_observations'] = convert_time_series_dataset_to_dict(observations_location_slice, decimals) + + for scenario in scenarios: + # Skip the scenario if it's not available for this variable + if f'{scenario}_{var}_p50' not in delta_30y_slice: + continue + chart_series[f"delta7100_{scenario}_median"] = convert_time_series_dataset_to_dict( + delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p50'], decimals) + chart_series[f"delta7100_{scenario}_range"] = convert_time_series_dataset_to_dict( + xr.merge([delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p10'], + delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p90']]), decimals) + + if delta_30y_slice[f'{scenarios[0]}_{var}_p50'].attrs.get('units') == 'K': + delta_30y_slice = delta_30y_slice + app.config['KELVIN_TO_C'] + + for scenario in scenarios: + # Skip the scenario if it's not available for this variable + if f'{scenario}_{var}_p50' not in delta_30y_slice: + continue + chart_series[f"30y_{scenario}_median"] = convert_time_series_dataset_to_dict( + delta_30y_slice[f'{scenario}_{var}_p50'], + decimals) + chart_series[f"30y_{scenario}_range"] = convert_time_series_dataset_to_dict( + xr.merge([delta_30y_slice[f'{scenario}_{var}_p10'], + delta_30y_slice[f'{scenario}_{var}_p90']]), decimals) + + return chart_series + + def generate_charts(var, lat, lon, month='ann'): """ Rewrite of get_values, generating a JSON ready for highcharts @@ -127,16 +169,23 @@ def generate_charts(var, lat, lon, month='ann'): observations_location_slice = None observations_dataset = None - bccaq_dataset = open_dataset(dataset_name, 'allyears', var, msys, monthpath) - bccaq_location_slice = bccaq_dataset.sel(lon=loni, lat=lati, method='nearest').drop(['lat', 'lon']).dropna('time') - - delta_30y_dataset = bccaq_dataset = open_dataset(dataset_name, '30ygraph', var, msys, monthpath) + delta_30y_dataset = open_dataset(dataset_name, '30ygraph', var, msys, monthpath) delta_30y_slice = delta_30y_dataset.sel(lon=loni, lat=lati, method='nearest').drop( [i for i in delta_30y_dataset.coords if i != 'time']).dropna('time') - chart_series = _format_slices_to_highcharts_series(observations_location_slice, bccaq_location_slice, delta_30y_slice, - var, decimals, dataset_name) - bccaq_dataset.close() + if var.startswith('rl'): # return period variables + if dataset_name != 'CMIP6': + return f"Bad request : return period variables only use the CMIP6 dataset, and has no {dataset_name} data available.\n", 400 + chart_series = _format_slices_to_highcharts_series_return_periods( + observations_location_slice, delta_30y_slice, var, decimals, dataset_name) + else: + bccaq_dataset = open_dataset(dataset_name, 'allyears', var, msys, monthpath) + bccaq_location_slice = bccaq_dataset.sel(lon=loni, lat=lati, method='nearest').drop(['lat', 'lon']).dropna('time') + + chart_series = _format_slices_to_highcharts_series(observations_location_slice, bccaq_location_slice, delta_30y_slice, + var, decimals, dataset_name) + bccaq_dataset.close() + if observations_dataset: observations_dataset.close() delta_30y_dataset.close() diff --git a/default_settings.py b/default_settings.py index b4a551d..34f9d00 100644 --- a/default_settings.py +++ b/default_settings.py @@ -125,6 +125,7 @@ 'HXmax30', 'HXmax35', 'HXmax40', + # Snow/rain fall 'first_snowfall', 'last_snowfall', 'snowfall_season_length', @@ -134,6 +135,25 @@ 'sn10mm', 'ratot', 'rax1day', + # Return periods + 'rl10pr', + 'rl10tasmax', + 'rl10tasmin', + 'rl20pr', + 'rl20tasmax', + 'rl20tasmin', + 'rl25pr', + 'rl25tasmax', + 'rl25tasmin', + 'rl30pr', + 'rl30tasmax', + 'rl30tasmin', + 'rl50pr', + 'rl50tasmax', + 'rl50tasmin', + 'rl5pr', + 'rl5tasmax', + 'rl5tasmin', ] SPEI_VARIABLES = ['spei_3m', 'spei_12m'] From ab234b9f0995ddcda4935e1c84db571588410f2f Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Wed, 13 May 2026 18:04:16 -0400 Subject: [PATCH 2/7] implement regional charts for return periods --- climatedata_api/charts.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index e3bc47b..0fed72c 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -333,26 +333,33 @@ def generate_regional_charts(partition, index, var, month='ann'): observations_dataset = None observations_location_slice = None - bccaq_dataset = open_dataset(dataset_name, 'allyears', var, msys, partition=partition) - bccaq_location_slice = bccaq_dataset.sel(geom=indexi).drop( - [i for i in bccaq_dataset.coords if i != 'time']).dropna('time') - delta_30y_dataset = open_dataset(dataset_name, '30ygraph', var, msys, partition=partition) delta_30y_slice = delta_30y_dataset.sel(geom=indexi).drop( [i for i in delta_30y_dataset.coords if i != 'time']).dropna('time') - # we filter the appropriate month/season from the MS or QS-DEC file - if msys in ["MS", "QS-DEC"]: - bccaq_location_slice = bccaq_location_slice.sel(time=(bccaq_location_slice.time.dt.month == monthnumber)) - if observations_location_slice: - observations_location_slice = observations_location_slice.sel( - time=(observations_location_slice.time.dt.month == monthnumber)) - delta_30y_slice = delta_30y_slice.sel( - time=(delta_30y_slice.time.dt.month == monthnumber)) - - chart_series = _format_slices_to_highcharts_series(observations_location_slice, bccaq_location_slice, delta_30y_slice, - var, decimals, dataset_name) - bccaq_dataset.close() + if var.startswith('rl'): # return period variables + if dataset_name != 'CMIP6': + return f"Bad request : return period variables only use the CMIP6 dataset, and has no {dataset_name} data available.\n", 400 + chart_series = _format_slices_to_highcharts_series_return_periods( + observations_location_slice, delta_30y_slice, var, decimals, dataset_name) + else: + bccaq_dataset = open_dataset(dataset_name, 'allyears', var, msys, partition=partition) + bccaq_location_slice = bccaq_dataset.sel(geom=indexi).drop( + [i for i in bccaq_dataset.coords if i != 'time']).dropna('time') + + # we filter the appropriate month/season from the MS or QS-DEC file + if msys in ["MS", "QS-DEC"]: + bccaq_location_slice = bccaq_location_slice.sel(time=(bccaq_location_slice.time.dt.month == monthnumber)) + if observations_location_slice: + observations_location_slice = observations_location_slice.sel( + time=(observations_location_slice.time.dt.month == monthnumber)) + delta_30y_slice = delta_30y_slice.sel( + time=(delta_30y_slice.time.dt.month == monthnumber)) + + chart_series = _format_slices_to_highcharts_series(observations_location_slice, bccaq_location_slice, delta_30y_slice, + var, decimals, dataset_name) + bccaq_dataset.close() + if observations_dataset: observations_dataset.close() delta_30y_dataset.close() From 175b6b27237801846b273cec235e736eff6f2d17 Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Thu, 14 May 2026 09:28:16 -0400 Subject: [PATCH 3/7] add checks for return period variables for the download endpoint --- climatedata_api/download.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/climatedata_api/download.py b/climatedata_api/download.py index a75d909..0480df7 100644 --- a/climatedata_api/download.py +++ b/climatedata_api/download.py @@ -338,6 +338,14 @@ def download(): monthnumber = app.config['MONTH_NUMBER_LUT'][month] datasets[0] = datasets[0].sel(time=(datasets[0].time.dt.month == monthnumber)) limit = app.config['SPEI_DATE_LIMIT'] + elif var.startswith('rl'): # return period variables + if dataset_name != 'CMIP6': + return f"Bad request : `{var}` variable only uses the CMIP6 dataset, and has no {dataset_name} data available.\n", 400 + if dataset_type != "30ygraph": + return f"Bad request : `{var}` variable only has 30-year averages, use the `30ygraph` dataset_type parameter.\n", 400 + if month != "ann": + return f"Bad request : `{var}` variable only supports the annual frequency, use the `ann` month parameter.\n", 400 + datasets = [open_dataset(dataset_name, dataset_type, var, freq, monthpath)] else: if month == 'all': datasets = [open_dataset(dataset_name, dataset_type, var, freq, m) for m in app.config['ALLMONTHS']] From a1239a7d45721bb460381f8522e0abc75031a516 Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Thu, 4 Jun 2026 16:09:27 -0400 Subject: [PATCH 4/7] add data for chart data points --- climatedata_api/charts.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index 0fed72c..62d07db 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -99,6 +99,15 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli # Skip the scenario if it's not available for this variable if f'{scenario}_{var}_p50' not in delta_30y_slice: continue + + # Add chart data points + chart_series[scenario + '_median'] = convert_time_series_dataset_to_list( + delta_30y_slice[f'{scenario}_{var}_p50'], decimals) + chart_series[scenario + '_range'] = convert_time_series_dataset_to_list( + xr.merge([delta_30y_slice[f'{scenario}_{var}_p10'], + delta_30y_slice[f'{scenario}_{var}_p90']]), decimals) + + # Add 30-year changes chart_series[f"delta7100_{scenario}_median"] = convert_time_series_dataset_to_dict( delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p50'], decimals) chart_series[f"delta7100_{scenario}_range"] = convert_time_series_dataset_to_dict( @@ -108,6 +117,7 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli if delta_30y_slice[f'{scenarios[0]}_{var}_p50'].attrs.get('units') == 'K': delta_30y_slice = delta_30y_slice + app.config['KELVIN_TO_C'] + # Add 30-year values for scenario in scenarios: # Skip the scenario if it's not available for this variable if f'{scenario}_{var}_p50' not in delta_30y_slice: From 20d2301aaadd584e4ceb166538970502e3c683c8 Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Fri, 5 Jun 2026 09:29:48 -0400 Subject: [PATCH 5/7] add year offset for return period data --- climatedata_api/charts.py | 17 ++++++++++------- climatedata_api/utils.py | 12 ++++++------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index 62d07db..9c5a515 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -87,13 +87,16 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli """ scenarios = app.config['SCENARIOS'][dataset_name] delta_naming = app.config['DELTA_NAMING'][dataset_name] + # Offset return period by 15 years (half of 30 year period), + # to have data points displayed at the middle of the 30y interval on the charts + year_offset = 15 chart_series = {} if observations_location_slice[var].attrs.get('units') == 'K': observations_location_slice = observations_location_slice + app.config['KELVIN_TO_C'] - chart_series['30y_observations'] = convert_time_series_dataset_to_dict(observations_location_slice, decimals) + chart_series['30y_observations'] = convert_time_series_dataset_to_dict(observations_location_slice, decimals, year_offset) for scenario in scenarios: # Skip the scenario if it's not available for this variable @@ -102,17 +105,17 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli # Add chart data points chart_series[scenario + '_median'] = convert_time_series_dataset_to_list( - delta_30y_slice[f'{scenario}_{var}_p50'], decimals) + delta_30y_slice[f'{scenario}_{var}_p50'], decimals, year_offset) chart_series[scenario + '_range'] = convert_time_series_dataset_to_list( xr.merge([delta_30y_slice[f'{scenario}_{var}_p10'], - delta_30y_slice[f'{scenario}_{var}_p90']]), decimals) + delta_30y_slice[f'{scenario}_{var}_p90']]), decimals, year_offset) # Add 30-year changes chart_series[f"delta7100_{scenario}_median"] = convert_time_series_dataset_to_dict( - delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p50'], decimals) + delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p50'], decimals, year_offset) chart_series[f"delta7100_{scenario}_range"] = convert_time_series_dataset_to_dict( xr.merge([delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p10'], - delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p90']]), decimals) + delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p90']]), decimals, year_offset) if delta_30y_slice[f'{scenarios[0]}_{var}_p50'].attrs.get('units') == 'K': delta_30y_slice = delta_30y_slice + app.config['KELVIN_TO_C'] @@ -124,10 +127,10 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli continue chart_series[f"30y_{scenario}_median"] = convert_time_series_dataset_to_dict( delta_30y_slice[f'{scenario}_{var}_p50'], - decimals) + decimals, year_offset) chart_series[f"30y_{scenario}_range"] = convert_time_series_dataset_to_dict( xr.merge([delta_30y_slice[f'{scenario}_{var}_p10'], - delta_30y_slice[f'{scenario}_{var}_p90']]), decimals) + delta_30y_slice[f'{scenario}_{var}_p90']]), decimals, year_offset) return chart_series diff --git a/climatedata_api/utils.py b/climatedata_api/utils.py index 0953f55..7e679d0 100644 --- a/climatedata_api/utils.py +++ b/climatedata_api/utils.py @@ -57,7 +57,7 @@ def open_dataset_by_path(path): raise FileNotFoundError(f"Dataset file not found: {path}") -def convert_time_series_dataset_to_list(dataset, decimals): +def convert_time_series_dataset_to_list(dataset, decimals, year_offset: int=0): """ Converts xarray dataset to a list. We assume that the coordinates are timestamps, which are converted to milliseconds since 1970-01-01 (integer) @@ -67,11 +67,11 @@ def convert_time_series_dataset_to_list(dataset, decimals): def _convert(float_list): return float_list if decimals > 0 else list(map(int, float_list)) - return [[int(a[0].timestamp() * 1000)] + _convert(a[1:]) for a in - dataset.to_dataframe().astype('float64').round(decimals).reset_index().values.tolist()] + df_list = dataset.to_dataframe().astype('float64').round(decimals).reset_index().values.tolist() + return [[int(a[0].replace(year=a[0].year + year_offset).timestamp() * 1000)] + _convert(a[1:]) for a in df_list] -def convert_time_series_dataset_to_dict(dataset, decimals): +def convert_time_series_dataset_to_dict(dataset, decimals, year_offset: int=0): """ Converts xarray dataset to a dict. We assume that the coordinates are timestamps, which are converted to milliseconds since 1970-01-01 (integer) @@ -81,8 +81,8 @@ def convert_time_series_dataset_to_dict(dataset, decimals): def _convert(float_list): return float_list if decimals > 0 else list(map(int, float_list)) - return {int(a[0].timestamp() * 1000): _convert(a[1:]) for a in - dataset.to_dataframe().astype('float64').round(decimals).reset_index().values.tolist()} + df_list = dataset.to_dataframe().astype('float64').round(decimals).reset_index().values.tolist() + return {int(a[0].replace(year=a[0].year + year_offset).timestamp() * 1000): _convert(a[1:]) for a in df_list} SAFE_CHARACTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-" # Safe for URL From 140e95912a538debd6084cb7e97f1438eec3231c Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Fri, 5 Jun 2026 13:28:11 -0400 Subject: [PATCH 6/7] add missing 'observations' field from return period chart data --- climatedata_api/charts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index 9c5a515..8dbf7c3 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -96,6 +96,7 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli if observations_location_slice[var].attrs.get('units') == 'K': observations_location_slice = observations_location_slice + app.config['KELVIN_TO_C'] + chart_series['observations'] = convert_time_series_dataset_to_list(observations_location_slice, decimals, year_offset) chart_series['30y_observations'] = convert_time_series_dataset_to_dict(observations_location_slice, decimals, year_offset) for scenario in scenarios: From 3edbdd0aa5cede66d3022b260c49722652961898 Mon Sep 17 00:00:00 2001 From: Charles-William Cummings Date: Fri, 5 Jun 2026 14:56:24 -0400 Subject: [PATCH 7/7] code cleanup --- climatedata_api/charts.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/climatedata_api/charts.py b/climatedata_api/charts.py index 8dbf7c3..c233900 100644 --- a/climatedata_api/charts.py +++ b/climatedata_api/charts.py @@ -91,13 +91,10 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli # to have data points displayed at the middle of the 30y interval on the charts year_offset = 15 - chart_series = {} - - if observations_location_slice[var].attrs.get('units') == 'K': - observations_location_slice = observations_location_slice + app.config['KELVIN_TO_C'] - - chart_series['observations'] = convert_time_series_dataset_to_list(observations_location_slice, decimals, year_offset) - chart_series['30y_observations'] = convert_time_series_dataset_to_dict(observations_location_slice, decimals, year_offset) + chart_series = { + 'observations': convert_time_series_dataset_to_list(observations_location_slice, decimals, year_offset), + '30y_observations': convert_time_series_dataset_to_dict(observations_location_slice, decimals, year_offset) + } for scenario in scenarios: # Skip the scenario if it's not available for this variable @@ -118,9 +115,6 @@ def _format_slices_to_highcharts_series_return_periods(observations_location_sli xr.merge([delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p10'], delta_30y_slice[f'{scenario}_{var}_{delta_naming}_p90']]), decimals, year_offset) - if delta_30y_slice[f'{scenarios[0]}_{var}_p50'].attrs.get('units') == 'K': - delta_30y_slice = delta_30y_slice + app.config['KELVIN_TO_C'] - # Add 30-year values for scenario in scenarios: # Skip the scenario if it's not available for this variable