From 02405e79d91dfe8374bcd8d2e1f355587d633bec Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Sun, 30 Nov 2025 13:30:11 +1100 Subject: [PATCH 01/12] add battery storageunit basic implementation --- src/ispypsa/iasr_table_caching/local_cache.py | 3 + src/ispypsa/pypsa_build/build.py | 6 +- src/ispypsa/pypsa_build/carriers.py | 11 +- src/ispypsa/pypsa_build/storage.py | 48 ++ src/ispypsa/templater/create_template.py | 6 + src/ispypsa/templater/helpers.py | 20 +- src/ispypsa/templater/lists.py | 37 + src/ispypsa/templater/mappings.py | 116 ++- .../static_new_generator_properties.py | 7 +- src/ispypsa/templater/storage.py | 697 ++++++++++++++++++ .../translator/create_pypsa_friendly.py | 34 + src/ispypsa/translator/mappings.py | 48 ++ src/ispypsa/translator/storage.py | 295 ++++++++ 13 files changed, 1312 insertions(+), 16 deletions(-) create mode 100644 src/ispypsa/pypsa_build/storage.py create mode 100644 src/ispypsa/templater/storage.py create mode 100644 src/ispypsa/translator/storage.py diff --git a/src/ispypsa/iasr_table_caching/local_cache.py b/src/ispypsa/iasr_table_caching/local_cache.py index 5608f2d4..660da18a 100644 --- a/src/ispypsa/iasr_table_caching/local_cache.py +++ b/src/ispypsa/iasr_table_caching/local_cache.py @@ -80,6 +80,8 @@ "technology_specific_lcfs", ] + _GENERATOR_PROPERTY_TABLES +_BATTERY_REQUIRED_PROPERTY_TABLES = ["battery_properties"] + _POLICY_REQUIRED_TABLES = [ "vic_renewable_target_trajectory", "qld_renewable_target_trajectory", @@ -97,6 +99,7 @@ _NETWORK_REQUIRED_TABLES + _GENERATORS_STORAGE_REQUIRED_SUMMARY_TABLES + _GENERATORS_REQUIRED_PROPERTY_TABLES + + _BATTERY_REQUIRED_PROPERTY_TABLES + _NEW_ENTRANTS_COST_TABLES + _POLICY_REQUIRED_TABLES ) diff --git a/src/ispypsa/pypsa_build/build.py b/src/ispypsa/pypsa_build/build.py index b05c66f9..8f10f44e 100644 --- a/src/ispypsa/pypsa_build/build.py +++ b/src/ispypsa/pypsa_build/build.py @@ -15,6 +15,7 @@ from ispypsa.pypsa_build.initialise import _initialise_network from ispypsa.pypsa_build.investment_period_weights import _add_investment_period_weights from ispypsa.pypsa_build.links import _add_links_to_network +from ispypsa.pypsa_build.storage import _add_batteries_to_network def build_pypsa_network( @@ -58,7 +59,9 @@ def build_pypsa_network( network, pypsa_friendly_tables["investment_period_weights"] ) - _add_carriers_to_network(network, pypsa_friendly_tables["generators"]) + _add_carriers_to_network( + network, pypsa_friendly_tables["generators"], pypsa_friendly_tables["batteries"] + ) _add_buses_to_network( network, pypsa_friendly_tables["buses"], path_to_pypsa_friendly_timeseries_data @@ -72,6 +75,7 @@ def build_pypsa_network( pypsa_friendly_tables["generators"], path_to_pypsa_friendly_timeseries_data, ) + _add_batteries_to_network(network, pypsa_friendly_tables["batteries"]) if "custom_constraints_generators" in pypsa_friendly_tables.keys(): _add_bus_for_custom_constraints(network) diff --git a/src/ispypsa/pypsa_build/carriers.py b/src/ispypsa/pypsa_build/carriers.py index 9ce952e0..4618c474 100644 --- a/src/ispypsa/pypsa_build/carriers.py +++ b/src/ispypsa/pypsa_build/carriers.py @@ -4,15 +4,22 @@ import pypsa -def _add_carriers_to_network(network: pypsa.Network, generators: pd.DataFrame) -> None: +def _add_carriers_to_network( + network: pypsa.Network, generators: pd.DataFrame, storage: pd.DataFrame +) -> None: """Adds the Carriers in the generators table, and the AC and DC Carriers to the `pypsa.Network`. Args: network: The `pypsa.Network` object generators: `pd.DataFrame` with `PyPSA` style `Generator` attributes. + storage: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes. At the moment this comprises batteries only. Returns: None """ - carriers = list(generators["carrier"].unique()) + ["AC", "DC"] + carriers = ( + list(generators["carrier"].unique()) + + list(storage["carrier"].unique()) + + ["AC", "DC"] + ) network.add("Carrier", carriers) diff --git a/src/ispypsa/pypsa_build/storage.py b/src/ispypsa/pypsa_build/storage.py new file mode 100644 index 00000000..83d559b5 --- /dev/null +++ b/src/ispypsa/pypsa_build/storage.py @@ -0,0 +1,48 @@ +import pandas as pd +import pypsa + + +def _add_battery_to_network( + battery_definition: dict, + network: pypsa.Network, +) -> None: + """Adds a battery to a pypsa.Network based on a dict containing PyPSA StorageUnit + attributes. + + PyPSA StorageUnits have set power to energy capacity ratio, + + Args: + battery_definition: dict containing pypsa StorageUnit parameters + network: The `pypsa.Network` object + + Returns: None + """ + battery_definition["class_name"] = "StorageUnit" + + pypsa_attributes_only = { + key: value + for key, value in battery_definition.items() + if not key.startswith("isp_") + } + network.add(**pypsa_attributes_only) + + +def _add_batteries_to_network( + network: pypsa.Network, + batterys: pd.DataFrame, +) -> None: + """Adds the batterys in a pypsa-friendly `pd.DataFrame` to the `pypsa.Network`. + + Args: + network: The `pypsa.Network` object + batterys: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes. + Returns: None + """ + + batterys.apply( + lambda row: _add_battery_to_network( + row.to_dict(), + network, + ), + axis=1, + ) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index 343aa9dd..fea7ece8 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -29,6 +29,7 @@ from ispypsa.templater.static_new_generator_properties import ( _template_new_generators_static_properties, ) +from ispypsa.templater.storage import _template_battery_properties _BASE_TEMPLATE_OUTPUTS = [ "sub_regions", @@ -37,6 +38,7 @@ "flow_paths", "ecaa_generators", "new_entrant_generators", + "batteries", "coal_prices", "gas_prices", "liquid_fuel_prices", @@ -182,6 +184,10 @@ def create_ispypsa_inputs_template( iasr_tables ) + ecaa_batteries, new_entrant_batteries = _template_battery_properties(iasr_tables) + template["ecaa_batteries"] = ecaa_batteries + template["new_entrant_batteries"] = new_entrant_batteries + dynamic_generator_property_templates = _template_generator_dynamic_properties( iasr_tables, scenario ) diff --git a/src/ispypsa/templater/helpers.py b/src/ispypsa/templater/helpers.py index 9e581902..4d6441a9 100644 --- a/src/ispypsa/templater/helpers.py +++ b/src/ispypsa/templater/helpers.py @@ -334,6 +334,9 @@ def _rez_name_to_id_mapping( ) -> pd.Series: """Maps REZ names to REZ IDs.""" + if series.empty or series is None or all(series.isna()): + return series + # add non-REZs to the REZ table and set up mapping: non_rez_ids = pd.DataFrame( { @@ -351,19 +354,24 @@ def _rez_name_to_id_mapping( # ------ clean up the series in case of old/unsupported REZ names # update references to "North [East|West] Tasmania Coast" to "North Tasmania Coast" # update references to "Portland Coast" to "Southern Ocean" - series = series.replace( + series_fixed_rez_names = series.replace( { r".+Tasmania Coast": "North Tasmania Coast", r"Portland Coast": "Southern Ocean", }, regex=True, ) - # fuzzy match series to REZ names to make sure they are consistent - series = _fuzzy_match_names( - series, - renewable_energy_zones["Name"], + # fuzzy match series to REZ names to make sure they are consistent - but only + # for not-exact matches that already exist, to avoid skipping necessary fixes: + where_not_existing_match_str = series_fixed_rez_names.apply( + lambda x: x not in rez_name_to_id.keys() + ) + + series_fixed_rez_names.loc[where_not_existing_match_str] = _fuzzy_match_names( + series_fixed_rez_names.loc[where_not_existing_match_str], + rez_name_to_id.keys(), f"mapping REZ names to REZ IDs for property '{series_name}'", threshold=90, ) - return series.replace(rez_name_to_id) + return series_fixed_rez_names.replace(rez_name_to_id) diff --git a/src/ispypsa/templater/lists.py b/src/ispypsa/templater/lists.py index a71f8c3b..5e181c53 100644 --- a/src/ispypsa/templater/lists.py +++ b/src/ispypsa/templater/lists.py @@ -13,6 +13,13 @@ "existing_committed_and_anticipated_batteries" ] +_ALL_GENERATOR_STORAGE_SUMMARIES = _ALL_GENERATOR_TYPES + ["batteries"] + +_ECAA_BATTERY_TYPES = [ + "existing_committed_and_anticipated_batteries", + "additional_projects", +] + _CONDENSED_GENERATOR_TYPES = [ "existing_committed_anticipated_additional_generators", "new_entrants", @@ -54,3 +61,33 @@ "minimum_stable_level_%", "minimum_load_mw", ] + + +_MINIMUM_REQUIRED_BATTERY_COLUMNS = [ + "storage_name", + "isp_resource_type", + "technology_type", + "status", + # region + "region_id", + "sub_region_id", + "rez_id", + # carrier + "fuel_type", + # capital cost input + "fom_$/kw/annum", + # connection/build cost & limits + "connection_cost_$/mw", + "technology_specific_lcf_%", + # capacity + "maximum_capacity_mw", + "storage_duration_hours", + # battery timing + "commissioning_date", + "closure_year", + "lifetime", + # operational + "round_trip_efficiency_%", + "charging_efficiency_%", + "discharging_efficiency_%", +] diff --git a/src/ispypsa/templater/mappings.py b/src/ispypsa/templater/mappings.py index b7e92429..1571e5e3 100644 --- a/src/ispypsa/templater/mappings.py +++ b/src/ispypsa/templater/mappings.py @@ -4,9 +4,9 @@ from .lists import ( _ALL_GENERATOR_STORAGE_TYPES, _CONDENSED_GENERATOR_TYPES, + _ECAA_BATTERY_TYPES, _ECAA_GENERATOR_TYPES, _ISP_SCENARIOS, - _NEW_GENERATOR_TYPES, ) _NEM_REGION_IDS = pd.Series( @@ -80,6 +80,31 @@ "summer_peak_rating_%": "summer_rating_mw", "technology_specific_lcf_%": "regional_build_cost_zone", } + +_ECAA_STORAGE_NEW_COLUMN_MAPPING = { + "maximum_capacity_mw": "storage_name", + "energy_capacity_mwh": "storage_name", + "storage_duration_hours": "storage_name", + "round_trip_efficiency_%": "fom_$/kw/annum", + "charging_efficiency_%": "fom_$/kw/annum", + "discharging_efficiency_%": "fom_$/kw/annum", + "lifetime": "storage_name", + "commissioning_date": "storage_name", + "closure_year": "storage_name", + "isp_resource_type": "fom_$/kw/annum", +} + +_NEW_STORAGE_NEW_COLUMN_MAPPING = { + "maximum_capacity_mw": "storage_name", + "storage_duration_hours": "storage_name", + "round_trip_efficiency_%": "storage_name", + "charging_efficiency_%": "storage_name", + "discharging_efficiency_%": "storage_name", + "lifetime": "storage_name", + "technology_specific_lcf_%": "regional_build_cost_zone", + "isp_resource_type": "storage_name", +} + """ _NEW_COLUMN_MAPPING dicts define new/additional columns to be added to the corresponding ECAA or new entrant generator summary tables. Keys are the name of the column to be added @@ -281,6 +306,95 @@ for opex mapping to rename columns in the table. """ +_ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP = { + "maximum_capacity_mw": dict( + table=[f"maximum_capacity_{gen_type}" for gen_type in _ECAA_BATTERY_TYPES], + table_lookup="Storage", + alternative_lookups=["Project"], + table_value="Installed capacity (MW)", + ), + "energy_capacity_mwh": dict( + table=[f"maximum_capacity_{gen_type}" for gen_type in _ECAA_BATTERY_TYPES], + table_lookup="Storage", + alternative_lookups=["Project"], + table_value="Energy (MWh)", + ), + "commissioning_date": dict( + table=[f"maximum_capacity_{gen_type}" for gen_type in _ECAA_BATTERY_TYPES], + table_lookup="Storage", + alternative_lookups=["Project"], + table_value="Indicative commissioning date", + ), + "fom_$/kw/annum": dict( + table="fixed_opex_existing_committed_anticipated_additional_generators", + table_lookup="Generator", + table_value="Fixed OPEX ($/kW/year)", + ), + "vom_$/mwh_sent_out": dict( + table="variable_opex_existing_committed_anticipated_additional_generators", + table_lookup="Generator", + table_value="Variable OPEX ($/MWh sent out)", + ), + "round_trip_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Round trip efficiency (utility)", + ), + "charging_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Charge efficiency (utility)", + ), + "discharging_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Discharge efficiency (utility)", + ), +} + +_NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP = { + "maximum_capacity_mw": dict( + table="maximum_capacity_new_entrants", + table_lookup="Generator type", + table_value="Total plant size (MW)", + ), + "fom_$/kw/annum": dict( + table="fixed_opex_new_entrants", + table_lookup="Generator", + table_col_prefix="Fixed OPEX ($/kW sent out/year)", + ), + "vom_$/mwh_sent_out": dict( + table="variable_opex_new_entrants", + table_lookup="Generator", + table_col_prefix="Variable OPEX ($/MWh sent out)", + ), + "lifetime": dict( + table="lead_time_and_project_life", + table_lookup="Technology", + table_value="Technical life (years) 6", + ), + "storage_duration_hours": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Energy capacity", + ), + "round_trip_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Round trip efficiency (utility)", + ), + "charging_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Charge efficiency (utility)", + ), + "discharging_efficiency_%": dict( + table="battery_properties", + table_lookup="storage_name", + table_value="Discharge efficiency (utility)", + ), +} + """ _TEMPLATE_RENEWABLE_ENERGY_TARGET_MAP is a dictionary that maps template functions to lists of dictionaries containing the CSV file name, region_id and policy_id for each diff --git a/src/ispypsa/templater/static_new_generator_properties.py b/src/ispypsa/templater/static_new_generator_properties.py index c42f277b..9a28a19b 100644 --- a/src/ispypsa/templater/static_new_generator_properties.py +++ b/src/ispypsa/templater/static_new_generator_properties.py @@ -140,9 +140,6 @@ def _fix_forced_outage_columns(df: pd.DataFrame) -> pd.DataFrame: # (AEMO 2024 | Appendix 3. Renewable Energy Zones, p.38) df = df.loc[~(df["rez_location"] == "Illawarra"), :] - # adds extra necessary columns taking appropriate mapping values - # NOTE: this could be done more efficiently in future if needed, potentially - # adding a `new_mapping` field to relevant table map dicts? for ( new_column, existing_column_mapping, @@ -179,9 +176,7 @@ def _merge_and_set_new_generators_static_properties( df, col = _process_and_merge_opex(df, data, col, table_attrs) else: if type(table_attrs["table"]) is list: - data = [ - iasr_tables[table_attrs["table"]] for table in table_attrs["table"] - ] + data = [iasr_tables[table] for table in table_attrs["table"]] data = pd.concat(data, axis=0) else: data = iasr_tables[table_attrs["table"]] diff --git a/src/ispypsa/templater/storage.py b/src/ispypsa/templater/storage.py new file mode 100644 index 00000000..6ffed65e --- /dev/null +++ b/src/ispypsa/templater/storage.py @@ -0,0 +1,697 @@ +# load in all the necessary summary tables: +# - batteries_summary +# - new_entrants_summary +# - additional_projects_summary + +# concat them together, filter for just battery only rows (simple) +# cut out the columns we don't need +# add necesary new columns +# merge in the info + +# key data to merge in: +# - efficiencies +# - cost related data? +# - lifetime +# - rez_id +# - max_hours (storage duration) + + +import logging +import re + +import pandas as pd + +from .helpers import ( + _fuzzy_match_names, + _one_to_one_priority_based_fuzzy_matching, + _rez_name_to_id_mapping, + _snakecase_string, + _standardise_storage_capitalisation, + _where_any_substring_appears, +) +from .lists import _ALL_GENERATOR_STORAGE_SUMMARIES, _MINIMUM_REQUIRED_BATTERY_COLUMNS +from .mappings import ( + _ECAA_STORAGE_NEW_COLUMN_MAPPING, + _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP, + _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP, + _NEW_STORAGE_NEW_COLUMN_MAPPING, +) + + +def _template_battery_properties( + iasr_tables: dict[str, pd.DataFrame], +) -> tuple[pd.DataFrame, pd.DataFrame]: + storage_summaries = [] + for gen_storage_type in _ALL_GENERATOR_STORAGE_SUMMARIES: + summary_df = iasr_tables[_snakecase_string(gen_storage_type) + "_summary"] + summary_df.columns = ["Storage Name", *summary_df.columns[1:]] + storage_summaries.append(summary_df) + + storage_summaries = pd.concat(storage_summaries, axis=0).reset_index(drop=True) + + cleaned_storage_summaries = _clean_storage_summary(storage_summaries) + + # filter to just return "battery" rows for now + battery_storage_rows = cleaned_storage_summaries["technology_type"].str.contains( + r"battery", case=False + ) + # need to add pumped hydro properties to isp-workbook-parser to handle PHES + pumped_hydro_rows = cleaned_storage_summaries["technology_type"].str.contains( + r"pumped hydro", case=False + ) + + cleaned_battery_summaries = cleaned_storage_summaries[ + battery_storage_rows + ].reset_index(drop=True) + + # restructure the battery properties table for easier merging: + battery_properties = _restructure_battery_property_table( + iasr_tables["battery_properties"] + ) + iasr_tables["battery_properties"] = battery_properties + + # Separate out ECAA and new entrants again for some of the merging in of static properties + ecaa_battery_summary = cleaned_battery_summaries[ + cleaned_battery_summaries["status"].isin( + ["Existing", "Committed", "Anticipated", "Additional projects"] + ) + ].copy() + merged_cleaned_ecaa_battery_summaries = ( + _merge_and_set_ecaa_battery_static_properties(ecaa_battery_summary, iasr_tables) + ) + ecaa_required_cols = [ + col + for col in _MINIMUM_REQUIRED_BATTERY_COLUMNS + if col in merged_cleaned_ecaa_battery_summaries.columns + ] + + new_entrant_battery_summary = cleaned_battery_summaries[ + cleaned_battery_summaries["status"].isin(["New Entrant"]) + ].copy() + merged_cleaned_new_entrant_battery_summaries = ( + _merge_and_set_new_battery_static_properties( + new_entrant_battery_summary, iasr_tables + ) + ) + new_entrant_required_cols = [ + col + for col in _MINIMUM_REQUIRED_BATTERY_COLUMNS + if col in merged_cleaned_new_entrant_battery_summaries.columns + ] + + return ( + merged_cleaned_ecaa_battery_summaries[ecaa_required_cols], + merged_cleaned_new_entrant_battery_summaries[new_entrant_required_cols], + ) + + +def _clean_storage_summary(df: pd.DataFrame) -> pd.DataFrame: + """Cleans the storage summary table + + 1. Converts column names to snakecase + 2. Adds "_id" to the end of region/sub-region ID columns + 3. Removes redundant outage columns + 4. Enforces consistent formatting of "storage" str instances + + Args: + df: Storage summary `pd.DataFrame` + + Returns: + `pd.DataFrame`: Cleaned storage summary DataFrame + """ + + def _fix_forced_outage_columns(df: pd.DataFrame) -> pd.DataFrame: + """Removes until/post 2022 distinction in columns if it exists""" + if ( + any(until_cols := [col for col in df.columns if "until" in col]) + and any(post_cols := [col for col in df.columns if "post" in col]) + and len(until_cols) == len(post_cols) + ): + df = df.rename( + columns={col: col.replace("_until_2022", "") for col in until_cols} + ) + df = df.drop(columns=post_cols) + return df + + # clean up column naming + df.columns = [_snakecase_string(col_name) for col_name in df.columns] + df = df.rename( + columns={col: (col + "_id") for col in df.columns if re.search(r"region$", col)} + ) + df = _fix_forced_outage_columns(df) + + # standardise storage names + df["storage_name"] = _standardise_storage_capitalisation(df["storage_name"]) + return df + + +def _merge_and_set_ecaa_battery_static_properties( + df: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame] +): + """Merges into and sets static (i.e. not time-varying) storage unit properties in the + existing storage units template, and renames columns if this is specified + in the mapping. + + Uses `ispypsa.templater.mappings._ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP` + as the mapping. + + Args: + df: Existing storage summary DataFrame + iasr_tables: Dict of tables from the IASR workbook that have been parsed using + `isp-workbook-parser`. + + Returns: + `pd.DataFrame`: Existing storage template with static properties filled in + """ + + # add new columns with existing column mapping: + for new_column, existing_column_mapping in _ECAA_STORAGE_NEW_COLUMN_MAPPING.items(): + df[new_column] = df[existing_column_mapping] + + merged_static_cols = [] + for col, table_attrs in _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP.items(): + if type(table_attrs["table"]) is list: + data = [iasr_tables[table] for table in table_attrs["table"]] + data = pd.concat(data, axis=0) + else: + data = iasr_tables[table_attrs["table"]] + df, col = _merge_table_data(df, col, data, table_attrs) + merged_static_cols.append(col) + + df = _add_closure_year_column(df, iasr_tables["expected_closure_years"]) + df = _calculate_storage_duration_hours(df) + df = _add_and_clean_rez_ids(df, "rez_id", iasr_tables["renewable_energy_zones"]) + df = _add_isp_resource_type_column(df) + + # replace remaining string values in static property columns + df = df.infer_objects() + for col in [ + col + for col in merged_static_cols + if df[col].dtype == "object" + and "date" not in col # keep instances of date/datetime strings as strings + ]: + df[col] = df[col].apply(lambda x: pd.NA if isinstance(x, str) else x) + + return df + + +def _merge_and_set_new_battery_static_properties( + df: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame] +) -> pd.DataFrame: + """Merges into and sets static (i.e. not time-varying) storage unit properties for new + entrant storage units template, and renames columns if this is specified + in the mapping. + + Uses `ispypsa.templater.mappings._NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP` + as the mapping. + + Args: + df: New entrant storage summary DataFrame + iasr_tables: Dict of tables from the IASR workbook that have been parsed using + `isp-workbook-parser`. + + Returns: + `pd.DataFrame`: New entrant storage template with static properties filled in + """ + # add new columns with existing column mapping: + for new_column, existing_column_mapping in _NEW_STORAGE_NEW_COLUMN_MAPPING.items(): + df[new_column] = df[existing_column_mapping] + + # merge in static properties using the static property mapping + merged_static_cols = [] + for col, table_attrs in _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP.items(): + # if col is an opex column, use separate function to handle merging in: + if re.search("^[fv]om_", col): + data = iasr_tables[table_attrs["table"]] + df, col = _process_and_merge_opex(df, data, col, table_attrs) + else: + if type(table_attrs["table"]) is list: + data = [iasr_tables[table] for table in table_attrs["table"]] + data = pd.concat(data, axis=0) + else: + data = iasr_tables[table_attrs["table"]] + df, col = _merge_table_data(df, col, data, table_attrs) + merged_static_cols.append(col) + + df = _calculate_and_merge_tech_specific_lcfs( + df, iasr_tables, "technology_specific_lcf_%" + ) + df = _process_and_merge_connection_cost(df, iasr_tables["connection_costs_other"]) + df = _add_and_clean_rez_ids(df, "rez_id", iasr_tables["renewable_energy_zones"]) + df = _add_isp_resource_type_column(df) + df = _add_unique_new_entrant_storage_name_column(df) + + # replace remaining string values in static property columns + df = df.infer_objects() + for col in [col for col in merged_static_cols if df[col].dtype == "object"]: + df[col] = df[col].apply(lambda x: pd.NA if isinstance(x, str) else x) + return df + + +def _merge_table_data( + df: pd.DataFrame, col: str, table_data: pd.DataFrame, table_attrs: dict +) -> tuple[pd.DataFrame, str]: + """Replace values in the provided column of the summary mapping with those + in the table data using the provided attributes in + `_STORAGE_STATIC_PROPERTY_TABLE_MAP` + """ + # handle alternative lookup and value columns + for alt_attr in ("lookup", "value"): + if f"alternative_{alt_attr}s" in table_attrs.keys(): + table_col = table_attrs[f"table_{alt_attr}"] + for alt_col in table_attrs[f"alternative_{alt_attr}s"]: + table_data[table_col] = table_data[table_col].where( + pd.notna, table_data[alt_col] + ) + replacement_dict = ( + table_data.loc[:, [table_attrs["table_lookup"], table_attrs["table_value"]]] + .set_index(table_attrs["table_lookup"]) + .squeeze() + .to_dict() + ) + # handles slight difference in capitalisation + where_str = df[col].apply(lambda x: isinstance(x, str)) + df.loc[where_str, col] = _fuzzy_match_names( + df.loc[where_str, col], + replacement_dict.keys(), + f"merging in the storage static property {col}", + not_match="existing", + threshold=90, + ) + df[col] = df[col].replace(replacement_dict) + if "new_col_name" in table_attrs.keys(): + df = df.rename(columns={col: table_attrs["new_col_name"]}) + col = table_attrs["new_col_name"] + return df, col + + +def _process_and_merge_opex( + df: pd.DataFrame, + table_data: pd.DataFrame, + col_name: str, + table_attrs: dict, +) -> tuple[pd.DataFrame, str]: + """Processes and merges in fixed or variable OPEX values for new entrant storage. + + In v6.0 of the IASR workbook the base values for all OPEX are found in + the column "NSW Low" or the relevant table, all other values are calculated + from this base value multiplied by the O&M locational cost factor. This function + merges in the post-LCF calculated values provided in the IASR workbook. + """ + # update the mapping in this column to include storage name and the cost region initially given + df[col_name] = df["storage_name"] + " " + df[col_name] + table_data = table_data.rename( + columns={ + col: col.replace(f"{table_attrs['table_col_prefix']}_", "") + for col in table_data.columns + } + ) + opex_table = table_data.melt( + id_vars=[table_attrs["table_lookup"]], + var_name="cost_region", + value_name="opex_value", + ) + # add column with same storage + cost_region mapping as df[col_name]: + opex_table["mapping"] = ( + opex_table[table_attrs["table_lookup"]] + " " + opex_table["cost_region"] + ) + opex_replacement_dict = ( + opex_table[["mapping", "opex_value"]].set_index("mapping").squeeze().to_dict() + ) + # use fuzzy matching in case of slight differences in storage names: + where_str = df[col_name].apply(lambda x: isinstance(x, str)) + df.loc[where_str, col_name] = _fuzzy_match_names( + df.loc[where_str, col_name], + opex_replacement_dict.keys(), + f"merging in the new entrant storage static property {col_name}", + not_match="existing", + threshold=90, + ) + df[col_name] = df[col_name].replace(opex_replacement_dict) + return df, col_name + + +def _add_closure_year_column( + df: pd.DataFrame, closure_years: pd.DataFrame +) -> pd.DataFrame: + """Adds a column containing the expected closure year (calendar year) for ECAA storage units. + + Note: the IASR table specifies the expected closure years as calendar years, without + giving more detail about the expected closure month elsewhere. For now, this function + also makes the OPINIONATED choice to just return the year given by the table as the + closure year. + + Args: + df: `ISPyPSA` formatted pd.DataFrame detailing the ECAA storage units. + closure_years: pd.Dataframe containing the IASR table `expected_generator_closure_years` + for the ECAA storage units, by unit. Expects that closure years are given as integers. + + Returns: + `pd.DataFrame`: ECAA storage attributes table with additional 'closure_year' column. + """ + + default_closure_year = -1 + + # reformat closure_years table for clearer mapping: + closure_years.columns = [_snakecase_string(col) for col in closure_years.columns] + closure_years = closure_years.rename( + columns={ + "generator_name": "storage_name", + "expected_closure_year_calendar_year": "closure_year", + } + ) + + # process closure_years to get the earliest expected closure year for each storage: + closure_years = ( + closure_years.sort_values("closure_year", ascending=True) + .drop_duplicates(subset="storage_name", keep="first") + .dropna(subset="closure_year") + ) + closure_years_dict = closure_years.set_index("storage_name")[ + "closure_year" + ].to_dict() + + where_str = df["closure_year"].apply(lambda x: isinstance(x, str)) + df.loc[where_str, "closure_year"] = _fuzzy_match_names( + df.loc[where_str, "closure_year"], + closure_years_dict.keys(), + f"adding closure_year column to ecaa_storage_summary table", + not_match="existing", + threshold=85, + ) + # map rather than replace to pass default value for undefined closure years: + df["closure_year"] = df["closure_year"].map( + lambda closure_year: closure_years_dict.get(closure_year, default_closure_year) + ) + + return df + + +def _process_and_merge_connection_cost( + df: pd.DataFrame, connection_costs_table: pd.DataFrame +) -> pd.DataFrame: + """Process and merge connection cost data from IASR tables to new entrant storage template. + + The function processes the connection cost data by creating a new column + "connection_cost_$/mw" and mapping the connection cost technology to the + corresponding connection cost values in $/MW from the IASR tables. + + Args: + df (pd.DataFrame): dataframe containing `ISPyPSA` formatted new entrant storage summary + table. + connection_costs_table (pd.DataFrame): parsed IASR table containing non-VRE connection costs. + + Returns: + pd.DataFrame: dataframe containing `ISPyPSA` formatted new entrant storage summary table + with connection costs in $/MW merged in as new column named "connection_cost_$/mw". + """ + df["connection_cost_$/mw"] = ( + df["connection_cost_rez/_region_id"] + "_" + df["connection_cost_technology"] + ) + for col in connection_costs_table.columns[1:]: + connection_costs_table[col] *= 1000 # convert to $/mw + + connection_costs = ( + connection_costs_table.melt( + id_vars=["Region"], + var_name="connection_cost_technology", + value_name="connection_cost_$/mw", + ) + .fillna(0.0) + .reset_index() + ) + + battery_connection_costs = connection_costs[ + connection_costs["connection_cost_technology"].str.contains("Battery") + ].copy() + + battery_connection_costs["region_technology_mapping"] = ( + battery_connection_costs["Region"] + + "_" + + battery_connection_costs["connection_cost_technology"] + ) + battery_connection_costs_mapping = ( + battery_connection_costs[["region_technology_mapping", "connection_cost_$/mw"]] + .set_index("region_technology_mapping") + .squeeze() + .to_dict() + ) + + where_str = df["connection_cost_$/mw"].apply(lambda x: isinstance(x, str)) + df.loc[where_str, "connection_cost_$/mw"] = _fuzzy_match_names( + df.loc[where_str, "connection_cost_$/mw"], + battery_connection_costs_mapping.keys(), + "merging in the new entrant battery static property 'connection_cost_$/mw'", + not_match="existing", + threshold=90, + ) + + df["connection_cost_$/mw"] = df["connection_cost_$/mw"].replace( + battery_connection_costs_mapping + ) + missing_connection_costs = df["connection_cost_$/mw"].map( + lambda x: not isinstance(x, (float, int)) + ) + batteries_missing_connection_costs = df.loc[missing_connection_costs] + if not batteries_missing_connection_costs.empty: + raise ValueError( + f"Missing connection costs for the following batteries: {batteries_missing_connection_costs['storage_name'].unique()}" + ) + + return df + + +def _calculate_and_merge_tech_specific_lcfs( + df: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame], tech_lcf_col: str +) -> pd.DataFrame: + """Calculates the technology-specific locational cost factor as a percentage + for each new entrant storage unit and merges into summary mapping table. + """ + # loads in the three tables needed + breakdown_ratios = iasr_tables["technology_cost_breakdown_ratios"].reset_index() + breakdown_ratios = breakdown_ratios.loc[ + _where_any_substring_appears(breakdown_ratios["Technology"], ["battery"]) + ].copy() + technology_specific_lcfs = iasr_tables["technology_specific_lcfs"] + # loads all cols unless the str "O&M" is in col name + locational_cost_factors = iasr_tables["locational_cost_factors"] + locational_cost_factors = locational_cost_factors.set_index( + locational_cost_factors.columns[0] + ) + cols = [col for col in locational_cost_factors.columns if "O&M" not in col] + locational_cost_factors = locational_cost_factors.loc[:, cols] + + # reshape technology_specific_lcfs and name columns manually: + technology_specific_lcfs = ( + technology_specific_lcfs.melt( + id_vars="Cost zones / Sub-region", value_name="LCF", var_name="Technology" + ) + .dropna(axis=0, how="any") + .reset_index(drop=True) + ) + technology_specific_lcfs = technology_specific_lcfs.loc[ + _where_any_substring_appears( + technology_specific_lcfs["Technology"], ["battery"] + ) + ].copy() + technology_specific_lcfs.rename( + columns={"Cost zones / Sub-region": "Location"}, inplace=True + ) + # ensures storage names in LCF tables match those in the summary table + for df_to_match_gen_names in [technology_specific_lcfs, breakdown_ratios]: + df_to_match_gen_names["Technology"] = _fuzzy_match_names( + df_to_match_gen_names["Technology"], + df["storage_name"].unique(), + "calculating and merging in LCFs to static new entrant storage summary", + not_match="existing", + threshold=90, + ) + df_to_match_gen_names.set_index("Technology", inplace=True) + # use fuzzy matching to ensure that col names in tables to combine match up: + fuzzy_column_renaming = _one_to_one_priority_based_fuzzy_matching( + set(locational_cost_factors.columns.to_list()), + set(breakdown_ratios.columns.to_list()), + not_match="existing", + threshold=90, + ) + locational_cost_factors.rename(columns=fuzzy_column_renaming, inplace=True) + # loops over rows to calculate LCF for batteries: + for tech, row in technology_specific_lcfs.iterrows(): + calculated_lcf = breakdown_ratios.loc[tech, :].dot( + locational_cost_factors.loc[row["Location"], :] + ) + calculated_lcf /= 100 + df.loc[ + ((df["storage_name"] == tech) & (df[tech_lcf_col] == row["Location"])), + tech_lcf_col, + ] = calculated_lcf + # fills rows with no LCF with pd.NA + df[tech_lcf_col] = df[tech_lcf_col].apply( + lambda x: pd.NA if isinstance(x, str) else x + ) + return df + + +def _calculate_storage_duration_hours(df: pd.DataFrame) -> pd.DataFrame: + """Calculates the storage duration (in hours) for each storage unit in the ECAA + storage attributes table. + + This function uses the `maximum_capacity_mw` and `energy_capacity_mwh` columns + in the summary table to calculate the storage duration in hours as + `energy_capacity_mwh`/`maximum_capacity_mw`. + + Requires that the columns `maximum_capacity_mw` and `energy_capacity_mwh` are + present and not NaN. If either of these columns is NaNfor a particular storage unit, + a warning will be logged and that storage unit will be dropped from the table. + + Returns: + pd.DataFrame: ECAA storage attributes table with the additional `storage_duration_hours` column. + """ + + def _safe_calculate_storage_duration_hours(row): + if pd.isna(row["maximum_capacity_mw"]) or pd.isna(row["energy_capacity_mwh"]): + logging.warning( + f"Could not calculate storage_duration_hours for {row['storage_name']} " + + "due to missing maximum_capacity_mw or energy_capacity_mwh value." + + "This storage unit will not be considered in the model." + ) + return pd.NA + elif row["maximum_capacity_mw"] == 0: + return 0 + else: + return row["energy_capacity_mwh"] / row["maximum_capacity_mw"] + + df["energy_capacity_mwh"] = pd.to_numeric( + df["energy_capacity_mwh"], errors="coerce" + ) + df["maximum_capacity_mw"] = pd.to_numeric( + df["maximum_capacity_mw"], errors="coerce" + ) + + df["storage_duration_hours"] = df.apply( + _safe_calculate_storage_duration_hours, axis=1 + ) + # drop rows with missing storage_duration_hours - these have been logged. + df = df.dropna(subset=["storage_duration_hours"], ignore_index=True) + + return df + + +def _restructure_battery_property_table( + battery_property_table: pd.DataFrame, +) -> pd.DataFrame: + """Restructures the IASR battery property table into a more usable format. + + The output table will have columns "storage_name" and the battery property names as + columns, with the values of those properties as the values in the table (converted to + numeric values where possible). Rows match storage names/mappings in the summary tables. + + Args: + battery_property_table: pd.DataFrame, `battery_properties` table from the IASR workbook. + + Returns: + pd.DataFrame, restructured battery property table. + """ + battery_properties = battery_property_table.set_index("Property") + battery_properties = battery_properties.T.reset_index(names="storage_name") + + battery_properties.columns.name = None + + columns_to_make_numeric = [ + col for col in battery_properties.columns if col != "storage_name" + ] + for col in columns_to_make_numeric: + battery_properties[col] = pd.to_numeric( + battery_properties[col], errors="coerce" + ) + + return battery_properties + + +def _add_and_clean_rez_ids( + df: pd.DataFrame, rez_id_col_name: str, renewable_energy_zones: pd.DataFrame +) -> pd.DataFrame: + """ + Merges REZ IDs into the new entrant storage table and cleans up REZ names. + + REZ IDs are unique letter/digit identifiers used in the IASR workbook. This function + also handles the Non-REZ IDs for Victoria (V0) and New South Wales (N0). There are + also some manual mappings to correct REZ names that have been updated/changed + across tables (currently in the IASR workbook v6.0): 'North [East/West] Tasmania Coast' + becomes 'North Tasmania Coast', 'Portland Coast' becomes 'Southern Ocean'. + + Args: + df: new entrant storage DataFrame + rez_id_col_name: str, name of the new column to be added. + renewable_energy_zones: a pd.Dataframe of the IASR table `renewable_energy_zones` + containing columns "ID" and "Name" used to map the REZ IDs. + + Returns: + pd.DataFrame: new entrant storage DataFrame with REZ ID column added. + """ + + # add a new column to hold the REZ IDs that maps to the current rez_location: + df[rez_id_col_name] = df["rez_location"] + + # update references to "North [East|West] Tasmania Coast" to "North Tasmania Coast" + # update references to "Portland Coast" to "Southern Ocean" + rez_or_region_cols = [col for col in df.columns if re.search(r"rez|region_id", col)] + + for col in rez_or_region_cols: + df[col] = _rez_name_to_id_mapping(df[col], col, renewable_energy_zones) + + return df + + +def _add_isp_resource_type_column(df: pd.DataFrame): + """Maps the 'isp_resource_type' column in the combined storage units template table + to a new column that holds a string describing the resource type. + + Uses a regular expression to extract the storage duration in hours from the + 'isp_resource_type' string, at the moment only battery storage is handled here so + the resulting string becomes 'Battery Storage {duration}h'. + + Args: + df: pd.DataFrame, storage units template table + + Returns: + pd.DataFrame: storage units template table with a new column + 'isp_resource_type' that holds the descriptive string. + """ + + def _get_storage_duration_for_battery_type(name: str) -> str | None: + duration_pattern = r"(?P\d+h)rs* storage" + duration_string = re.search(duration_pattern, name, re.IGNORECASE) + + if duration_string: + return "Battery Storage " + duration_string.group("duration") + else: + return None + + df["isp_resource_type"] = df["isp_resource_type"].map( + _get_storage_duration_for_battery_type + ) + + return df + + +def _add_unique_new_entrant_storage_name_column(df: pd.DataFrame): + """Adds a new column to the new entrant storage units template table to hold a unique + identifier for each storage unit. + + New entrant storage are not defined for each REZ, with sub-regions being the most + granular regional grouping as of IASR workbook v6.0. + + Args: + df: pd.DataFrame, new entrant storage units template table + + Returns: + pd.DataFrame: new entrant storage units template table with the "storage_name" column + filled by a unique identifier string for each row. + """ + + df["storage_name"] = df["isp_resource_type"] + "_" + df["sub_region_id"] + df["storage_name"] = df["storage_name"].map(_snakecase_string) + + return df diff --git a/src/ispypsa/translator/create_pypsa_friendly.py b/src/ispypsa/translator/create_pypsa_friendly.py index 731031a3..436a7ff8 100644 --- a/src/ispypsa/translator/create_pypsa_friendly.py +++ b/src/ispypsa/translator/create_pypsa_friendly.py @@ -36,6 +36,10 @@ _create_investment_period_weightings, create_pypsa_friendly_snapshots, ) +from ispypsa.translator.storage import ( + _translate_ecaa_batteries, + _translate_new_entrant_batteries, +) from ispypsa.translator.temporal_filters import _time_series_filter from ispypsa.translator.time_series_checker import _check_time_series @@ -45,6 +49,7 @@ "buses", "links", "generators", + "batteries", "custom_constraints_lhs", "custom_constraints_rhs", "custom_constraints_generators", @@ -121,6 +126,35 @@ def create_pypsa_friendly_inputs( # TODO: Log, improve error message (/ is this the right place for the error?) raise ValueError("No generator data returned from translator.") + batteries = [] + translated_ecaa_batteries = _translate_ecaa_batteries( + ispypsa_tables, + config.temporal.capacity_expansion.investment_periods, + config.network.nodes.regional_granularity, + config.network.nodes.rezs, + config.temporal.year_type, + ) + _append_if_not_empty(batteries, translated_ecaa_batteries) + + translated_new_entrant_batteries = _translate_new_entrant_batteries( + ispypsa_tables, + config.temporal.capacity_expansion.investment_periods, + config.discount_rate, + config.network.nodes.regional_granularity, + config.network.nodes.rezs, + ) + _append_if_not_empty(batteries, translated_new_entrant_batteries) + + if len(batteries) > 0: + pypsa_inputs["batteries"] = pd.concat( + batteries, + axis=0, + ignore_index=True, + ) + else: + # TODO: Log, improve error message + raise ValueError("No battery data returned from translator.") + buses = [] links = [] diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index 873fe200..b3d3e448 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -67,6 +67,54 @@ # that are used in calculating PyPSA input values for generators, but aren't # attributes of Generator objects and aren't passed to the network. +_ECAA_BATTERY_ATTRIBUTES = { + "storage_name": "name", + "maximum_capacity_mw": "p_nom", + "storage_duration_hours": "max_hours", + "p_nom_extendable": "p_nom_extendable", + "fuel_type": "carrier", + "commissioning_date": "build_year", + "lifetime": "lifetime", + "capital_cost": "capital_cost", + "charging_efficiency_%": "efficiency_store", + "discharging_efficiency_%": "efficiency_dispatch", + "rez_id": "isp_rez_id", + # isp_resource_type has a clear mapping of technology type and storage duration + "isp_resource_type": "isp_resource_type", +} + +_NEW_ENTRANT_BATTERY_ATTRIBUTES = { # attributes used by the PyPSA network model: + "storage_name": "name", + "p_nom": "p_nom", + "storage_duration_hours": "max_hours", + "p_nom_extendable": "p_nom_extendable", + "fuel_type": "carrier", + "build_year": "build_year", + "lifetime": "lifetime", + "capital_cost": "capital_cost", + "charging_efficiency_%": "efficiency_store", + "discharging_efficiency_%": "efficiency_dispatch", + # attributes used to filter/apply custom constraints: + "isp_resource_type": "isp_resource_type", + "rez_id": "isp_rez_id", + # keeping technology_type because it's not defined anywhere else for the ECAA generators and could be useful for plotting/labelling? + "technology_type": "isp_technology_type", +} + +_BATTERY_ATTRIBUTE_ORDER = [ + "name", + "bus", + "p_nom", + "max_hours", + "p_nom_extendable", + "carrier", + "build_year", + "lifetime", + "capital_cost", + "isp_resource_type", + "isp_rez_id", +] + _BUS_ATTRIBUTES = {"isp_sub_region_id": "name"} _LINK_ATTRIBUTES = { diff --git a/src/ispypsa/translator/storage.py b/src/ispypsa/translator/storage.py new file mode 100644 index 00000000..36dc4eb6 --- /dev/null +++ b/src/ispypsa/translator/storage.py @@ -0,0 +1,295 @@ +import re +from pathlib import Path +from typing import List, Literal + +import numpy as np +import pandas as pd +from isp_trace_parser import get_data + +from ispypsa.translator.helpers import ( + _add_investment_periods_as_build_years, + _annuitised_investment_costs, + _get_commissioning_or_build_year_as_int, + _get_financial_year_int_from_string, +) +from ispypsa.translator.mappings import ( + _BATTERY_ATTRIBUTE_ORDER, + _ECAA_BATTERY_ATTRIBUTES, + _NEW_ENTRANT_BATTERY_ATTRIBUTES, +) + + +def _translate_ecaa_batteries( + ispypsa_tables: dict[str, pd.DataFrame], + investment_periods: list[int], + regional_granularity: str = "sub_regions", + rez_handling: str = "discrete_nodes", + year_type: str = "fy", +) -> pd.DataFrame: + """Process data on existing, committed, anticipated, and additional (ECAA) batteries + into a format aligned with PyPSA inputs. + + Args: + ispypsa_tables: dictionary of dataframes providing the `ISPyPSA` input tables. + (add link to ispypsa input tables docs). + investment_periods: list of years in which investment periods start obtained + from the model configuration. + regional_granularity: Regional granularity of the nodes obtained from the model + configuration. Defaults to "sub_regions". + year_type: str which should be "fy" or "calendar". If "fy" then investment + period ints are interpreted as specifying financial years (according to the + calendar year the financial year ends in). + rez_handling: str from the model configuration that defines whether REZs are + modelled as distinct nodes, one of "discrete_nodes" or "attached_to_parent_node". Defaults to "discrete_nodes". + + Returns: + `pd.DataFrame`: `PyPSA` style ECAA battery attributes in tabular format. + """ + + ecaa_batteries = ispypsa_tables["ecaa_batteries"] + if ecaa_batteries.empty: + # TODO: log + # raise error? + return pd.DataFrame() + + # calculate lifetime based on expected closure_year - build_year: + ecaa_batteries["lifetime"] = ecaa_batteries["closure_year"].map( + lambda x: float(x - investment_periods[0]) if x > 0 else np.inf + ) + ecaa_batteries = ecaa_batteries[ecaa_batteries["lifetime"] > 0].copy() + + battery_attributes = _ECAA_BATTERY_ATTRIBUTES.copy() + # Decide which column to rename to be the bus column. + if regional_granularity == "sub_regions": + bus_column = "sub_region_id" + elif regional_granularity == "nem_regions": + bus_column = "region_id" + elif regional_granularity == "single_region": + # No existing column to use for bus, so create a new one. + ecaa_batteries["bus"] = "NEM" + bus_column = "bus" # Name doesn't need to change. + battery_attributes[bus_column] = "bus" + + if rez_handling == "discrete_nodes": + # make sure batteries are still connected to the REZ bus where applicable + rez_mask = ~ecaa_batteries["rez_id"].isna() + ecaa_batteries.loc[rez_mask, bus_column] = ecaa_batteries.loc[ + rez_mask, "rez_id" + ] + + ecaa_batteries["commissioning_date"] = ecaa_batteries["commissioning_date"].apply( + _get_commissioning_or_build_year_as_int, + default_build_year=investment_periods[0], + year_type=year_type, + ) + + ecaa_batteries["p_nom_extendable"] = False + ecaa_batteries["capital_cost"] = 0.0 + + # filter and rename columns according to PyPSA input names: + ecaa_batteries_pypsa_format = ecaa_batteries.loc[ + :, battery_attributes.keys() + ].rename(columns=battery_attributes) + + columns_in_order = [ + col + for col in _BATTERY_ATTRIBUTE_ORDER + if col in ecaa_batteries_pypsa_format.columns + ] + + return ecaa_batteries_pypsa_format[columns_in_order] + + +def _translate_new_entrant_batteries( + ispypsa_tables: dict[str, pd.DataFrame], + investment_periods: list[int], + wacc: float, + regional_granularity: str = "sub_regions", + rez_handling: str = "discrete_nodes", +) -> pd.DataFrame: + """Process data on new entrant batteries into a format aligned with `PyPSA` inputs. + + Args: + ispypsa_tables: dictionary of dataframes providing the `ISPyPSA` input tables. + (add link to ispypsa input tables docs). + investment_periods: list of years in which investment periods start obtained + from the model configuration. + wacc: as float, weighted average cost of capital, an interest rate specifying + how expensive it is to borrow money for the asset investment. + regional_granularity: Regional granularity of the nodes obtained from the model + configuration. Defaults to "sub_regions". + rez_handling: str from the model configuration that defines whether REZs are + modelled as distinct nodes, one of "discrete_nodes" or "attached_to_parent_node". Defaults to "discrete_nodes". + + Returns: + `pd.DataFrame`: `PyPSA` style new entrant battery attributes in tabular format. + """ + + new_entrant_batteries = ispypsa_tables["new_entrant_batteries"] + if new_entrant_batteries.empty: + # TODO: log + # raise error? + return pd.DataFrame() + + battery_attributes = _NEW_ENTRANT_BATTERY_ATTRIBUTES.copy() + # Decide which column to rename to be the bus column. + if regional_granularity == "sub_regions": + bus_column = "sub_region_id" + elif regional_granularity == "nem_regions": + bus_column = "region_id" + elif regional_granularity == "single_region": + # No existing column to use for bus, so create a new one. + new_entrant_batteries["bus"] = "NEM" + bus_column = "bus" # Name doesn't need to change. + battery_attributes[bus_column] = "bus" + + if rez_handling == "discrete_nodes": + # make sure batteries are still connected to the REZ bus where applicable + rez_mask = ~new_entrant_batteries["rez_id"].isna() + new_entrant_batteries.loc[rez_mask, bus_column] = new_entrant_batteries.loc[ + rez_mask, "rez_id" + ] + + # create a row for each new entrant battery in each possible build year (investment period) + new_entrant_batteries_all_build_years = _add_investment_periods_as_build_years( + new_entrant_batteries, investment_periods + ) + battery_df_with_build_costs = _add_new_entrant_battery_build_costs( + new_entrant_batteries_all_build_years, + ispypsa_tables["new_entrant_build_costs"], + ) + battery_df_with_capital_costs = ( + _calculate_annuitised_new_entrant_battery_capital_costs( + battery_df_with_build_costs, + wacc, + ) + ) + # nan capex -> build limit of 0.0MW (no build of this battery in this region allowed) + battery_df_with_capital_costs = battery_df_with_capital_costs[ + ~battery_df_with_capital_costs["capital_cost"].isna() + ].copy() + + # then add build_year to battery name to maintain unique battery ID column + battery_df_with_capital_costs["storage_name"] = ( + battery_df_with_capital_costs["storage_name"] + + "_" + + battery_df_with_capital_costs["build_year"].astype(str) + ) + + # add a p_nom column set to 0.0 for all new entrants: + battery_df_with_capital_costs["p_nom"] = 0.0 + + # Convert p_min_pu from percentage to float between 0-1: + battery_df_with_capital_costs["p_nom_extendable"] = True + + # filter and rename columns to PyPSA format + new_entrant_batteries_pypsa_format = battery_df_with_capital_costs.loc[ + :, battery_attributes.keys() + ].rename(columns=battery_attributes) + + columns_in_order = [ + col + for col in _BATTERY_ATTRIBUTE_ORDER + if col in new_entrant_batteries_pypsa_format.columns + ] + + return new_entrant_batteries_pypsa_format[columns_in_order] + + +def _add_new_entrant_battery_build_costs( + new_entrant_batteries: pd.DataFrame, + new_entrant_build_costs: pd.DataFrame, +) -> pd.DataFrame: + """ + Merge build costs into new_entrant_batteries table. + + Args: + new_entrant_batteries table: dataframe containing `ISPyPSA` formatted + new-entrant battery detail, with a row for each battery in every possible + build year. Must have column "build_year" with integer values. + new_entrant_build_costs: `ISPyPSA` formatted dataframe with build costs in $/MW for + each new-entrant battery type and build year. + + Returns: + pd.DataFrame: new_entrant_batteries table with build costs merged in as new column + called "build_cost_$/mw". + + Raises: + ValueError: if new_entrant_batteries table does not have column "build_year" or + if any new entrant batteries have build costs missing/undefined. + + Notes: + 1. The function assumes that new_entrant_build_costs has a "technology" column + that matches with the "technology_type" column in new_entrant_batteries table. + """ + + if "build_year" not in new_entrant_batteries.columns: + raise ValueError( + "new_entrant_batteries table must have column 'build_year' to merge in build costs." + ) + + build_costs = new_entrant_build_costs.melt( + id_vars=["technology"], var_name="build_year", value_name="build_cost_$/mw" + ).rename(columns={"technology": "technology_type"}) + # get the financial year int from build_year string: + build_costs["build_year"] = build_costs["build_year"].apply( + _get_financial_year_int_from_string, + args=("new entrant battery build costs", "fy"), + ) + # make sure new_entrant_batteries has build_year column with ints: + new_entrant_batteries["build_year"] = new_entrant_batteries["build_year"].astype( + "int64" + ) + # return battery table with build costs merged in + new_entrants_with_build_costs = new_entrant_batteries.merge(build_costs, how="left") + + # check for empty/undefined build costs: + undefined_build_cost_batteries = ( + new_entrants_with_build_costs[ + new_entrants_with_build_costs["build_cost_$/mw"].isna() + ]["technology_type"] + .unique() + .tolist() + ) + if undefined_build_cost_batteries: + raise ValueError( + f"Undefined build costs for new entrant batteries: {undefined_build_cost_batteries}" + ) + + return new_entrants_with_build_costs + + +def _calculate_annuitised_new_entrant_battery_capital_costs( + new_entrant_batteries: pd.DataFrame, + wacc: float, +) -> pd.DataFrame: + """Calculates annuitised capital cost of each new entrant battery in each possible + build year. + + Args: + new_entrant_batteries: dataframe containing `ISPyPSA` formatted + new-entrant battery detail, with a row for each battery in every possible + build year and sub-region. + wacc: as float, weighted average cost of capital, an interest rate specifying + how expensive it is to borrow money for the asset investment. + + Returns: + new_entrant_batteries: `ISPyPSA` formatted dataframe with new column + "capital_cost" containing the annuitised capital cost of each new entrant + battery in each possible build year. + """ + new_entrant_batteries["capital_cost"] = ( + new_entrant_batteries["build_cost_$/mw"] + * (new_entrant_batteries["technology_specific_lcf_%"] / 100) + + new_entrant_batteries["connection_cost_$/mw"] + ) + # annuitise: + new_entrant_batteries["capital_cost"] = new_entrant_batteries.apply( + lambda x: _annuitised_investment_costs(x["capital_cost"], wacc, x["lifetime"]), + axis=1, + ) + # add annual fixed opex (first converting to $/MW/annum) + new_entrant_batteries["capital_cost"] += ( + new_entrant_batteries["fom_$/kw/annum"] * 1000 + ) + return new_entrant_batteries From 9524eae60f8fe31f11ed8dd03de4bf57db7a035e Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 01:03:52 +1100 Subject: [PATCH 02/12] add new tests and update existing tests for battery implementation --- src/ispypsa/pypsa_build/build.py | 8 +- src/ispypsa/pypsa_build/carriers.py | 15 +- src/ispypsa/pypsa_build/storage.py | 18 +- src/ispypsa/templater/create_template.py | 3 +- src/ispypsa/templater/filter_template.py | 37 ++ src/ispypsa/templater/mappings.py | 10 - src/ispypsa/templater/storage.py | 129 ++-- .../translator/create_pypsa_friendly.py | 7 +- src/ispypsa/translator/mappings.py | 10 +- src/ispypsa/translator/storage.py | 35 +- tests/conftest.py | 28 +- tests/test_model/test_add_generators.py | 5 - tests/test_model/test_add_storage.py | 109 ++++ .../test_custom_constraints/batteries.csv | 3 + tests/test_templater/test_storage.py | 562 ++++++++++++++++++ .../test_translator/test_translate_storage.py | 355 +++++++++++ .../test_limit_on_transmission_expansion.py | 2 + .../test_time_varying_flow_path_costs.py | 2 + ...test_vre_build_limit_custom_constraints.py | 2 + .../battery_properties.csv | 7 + 20 files changed, 1220 insertions(+), 127 deletions(-) create mode 100644 tests/test_model/test_add_storage.py create mode 100644 tests/test_model/test_pypsa_friendly_inputs/test_custom_constraints/batteries.csv create mode 100644 tests/test_templater/test_storage.py create mode 100644 tests/test_translator/test_translate_storage.py create mode 100644 tests/test_workbook_table_cache/battery_properties.csv diff --git a/src/ispypsa/pypsa_build/build.py b/src/ispypsa/pypsa_build/build.py index 8f10f44e..68b0d0f6 100644 --- a/src/ispypsa/pypsa_build/build.py +++ b/src/ispypsa/pypsa_build/build.py @@ -60,7 +60,9 @@ def build_pypsa_network( ) _add_carriers_to_network( - network, pypsa_friendly_tables["generators"], pypsa_friendly_tables["batteries"] + network, + pypsa_friendly_tables.get("generators"), + pypsa_friendly_tables.get("batteries"), ) _add_buses_to_network( @@ -75,7 +77,9 @@ def build_pypsa_network( pypsa_friendly_tables["generators"], path_to_pypsa_friendly_timeseries_data, ) - _add_batteries_to_network(network, pypsa_friendly_tables["batteries"]) + + if "batteries" in pypsa_friendly_tables.keys(): + _add_batteries_to_network(network, pypsa_friendly_tables["batteries"]) if "custom_constraints_generators" in pypsa_friendly_tables.keys(): _add_bus_for_custom_constraints(network) diff --git a/src/ispypsa/pypsa_build/carriers.py b/src/ispypsa/pypsa_build/carriers.py index 4618c474..268662a9 100644 --- a/src/ispypsa/pypsa_build/carriers.py +++ b/src/ispypsa/pypsa_build/carriers.py @@ -17,9 +17,14 @@ def _add_carriers_to_network( Returns: None """ - carriers = ( - list(generators["carrier"].unique()) - + list(storage["carrier"].unique()) - + ["AC", "DC"] - ) + generator_carriers = [] + storage_carriers = [] + standard_carriers = ["AC", "DC"] + + if generators is not None and not generators.empty: + generator_carriers = list(generators["carrier"].unique()) + if storage is not None and not storage.empty: + storage_carriers = list(storage["carrier"].unique()) + + carriers = generator_carriers + storage_carriers + standard_carriers network.add("Carrier", carriers) diff --git a/src/ispypsa/pypsa_build/storage.py b/src/ispypsa/pypsa_build/storage.py index 83d559b5..3d564f2a 100644 --- a/src/ispypsa/pypsa_build/storage.py +++ b/src/ispypsa/pypsa_build/storage.py @@ -3,17 +3,15 @@ def _add_battery_to_network( - battery_definition: dict, network: pypsa.Network, + battery_definition: dict, ) -> None: - """Adds a battery to a pypsa.Network based on a dict containing PyPSA StorageUnit + """Adds a battery to a pypsa.Network based on a dict containing `PyPSA` `StorageUnit` attributes. - PyPSA StorageUnits have set power to energy capacity ratio, - Args: - battery_definition: dict containing pypsa StorageUnit parameters network: The `pypsa.Network` object + battery_definition: dict containing pypsa `StorageUnit` parameters Returns: None """ @@ -29,20 +27,20 @@ def _add_battery_to_network( def _add_batteries_to_network( network: pypsa.Network, - batterys: pd.DataFrame, + batteries: pd.DataFrame, ) -> None: - """Adds the batterys in a pypsa-friendly `pd.DataFrame` to the `pypsa.Network`. + """Adds the batteries in a pypsa-friendly `pd.DataFrame` to the `pypsa.Network` as `PyPSA` `StorageUnit`s. Args: network: The `pypsa.Network` object - batterys: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes. + batteries: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes. Returns: None """ - batterys.apply( + batteries.apply( lambda row: _add_battery_to_network( - row.to_dict(), network, + row.to_dict(), ), axis=1, ) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index fea7ece8..4b88dc3b 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -37,8 +37,9 @@ "renewable_energy_zones", "flow_paths", "ecaa_generators", + "ecaa_batteries", "new_entrant_generators", - "batteries", + "new_entrant_batteries", "coal_prices", "gas_prices", "liquid_fuel_prices", diff --git a/src/ispypsa/templater/filter_template.py b/src/ispypsa/templater/filter_template.py index 5b61e7e4..0a7459e7 100644 --- a/src/ispypsa/templater/filter_template.py +++ b/src/ispypsa/templater/filter_template.py @@ -72,6 +72,12 @@ def _filter_template( ) filtered.update(generator_dependent) + # Filter storage (battery) tables by same logic as generators: + battery_tables, all_selected_batteries = _filter_batteries( + template, selected_sub_regions + ) + filtered.update(battery_tables) + # Infer link names selected_link_names = _infer_link_names( filtered.get("flow_paths", pd.DataFrame()), @@ -273,6 +279,37 @@ def _filter_generator_dependent_tables( return filtered +def _filter_batteries( + template: dict[str, pd.DataFrame], + selected_sub_regions: list[str], +) -> Tuple[dict[str, pd.DataFrame], Set[str]]: + """Filter battery tables and return filtered tables and battery names.""" + filtered = {} + all_selected_batteries = set() + + # Filter ecaa_batteries + if "ecaa_batteries" in template: + filtered["ecaa_batteries"] = template["ecaa_batteries"][ + template["ecaa_batteries"]["sub_region_id"].isin(selected_sub_regions) + ].copy() + all_selected_batteries.update( + filtered["ecaa_batteries"]["storage_name"].unique() + ) + + # Filter new_entrant_batteries + if "new_entrant_batteries" in template: + filtered["new_entrant_batteries"] = template["new_entrant_batteries"][ + template["new_entrant_batteries"]["sub_region_id"].isin( + selected_sub_regions + ) + ].copy() + all_selected_batteries.update( + filtered["new_entrant_batteries"]["storage_name"].unique() + ) + + return filtered, all_selected_batteries + + def _get_selected_rezs(filtered_tables: dict[str, pd.DataFrame]) -> Set[str]: """Extract REZ IDs from filtered tables.""" if ( diff --git a/src/ispypsa/templater/mappings.py b/src/ispypsa/templater/mappings.py index 1571e5e3..a1d8cb9a 100644 --- a/src/ispypsa/templater/mappings.py +++ b/src/ispypsa/templater/mappings.py @@ -330,11 +330,6 @@ table_lookup="Generator", table_value="Fixed OPEX ($/kW/year)", ), - "vom_$/mwh_sent_out": dict( - table="variable_opex_existing_committed_anticipated_additional_generators", - table_lookup="Generator", - table_value="Variable OPEX ($/MWh sent out)", - ), "round_trip_efficiency_%": dict( table="battery_properties", table_lookup="storage_name", @@ -363,11 +358,6 @@ table_lookup="Generator", table_col_prefix="Fixed OPEX ($/kW sent out/year)", ), - "vom_$/mwh_sent_out": dict( - table="variable_opex_new_entrants", - table_lookup="Generator", - table_col_prefix="Variable OPEX ($/MWh sent out)", - ), "lifetime": dict( table="lead_time_and_project_life", table_lookup="Technology", diff --git a/src/ispypsa/templater/storage.py b/src/ispypsa/templater/storage.py index 6ffed65e..f113bcdf 100644 --- a/src/ispypsa/templater/storage.py +++ b/src/ispypsa/templater/storage.py @@ -1,21 +1,3 @@ -# load in all the necessary summary tables: -# - batteries_summary -# - new_entrants_summary -# - additional_projects_summary - -# concat them together, filter for just battery only rows (simple) -# cut out the columns we don't need -# add necesary new columns -# merge in the info - -# key data to merge in: -# - efficiencies -# - cost related data? -# - lifetime -# - rez_id -# - max_hours (storage duration) - - import logging import re @@ -37,6 +19,8 @@ _NEW_STORAGE_NEW_COLUMN_MAPPING, ) +pd.set_option("display.max_columns", None) + def _template_battery_properties( iasr_tables: dict[str, pd.DataFrame], @@ -86,7 +70,7 @@ def _template_battery_properties( ] new_entrant_battery_summary = cleaned_battery_summaries[ - cleaned_battery_summaries["status"].isin(["New Entrant"]) + cleaned_battery_summaries["status"] == "New Entrant" ].copy() merged_cleaned_new_entrant_battery_summaries = ( _merge_and_set_new_battery_static_properties( @@ -145,6 +129,39 @@ def _fix_forced_outage_columns(df: pd.DataFrame) -> pd.DataFrame: return df +def _restructure_battery_property_table( + battery_property_table: pd.DataFrame, +) -> pd.DataFrame: + """Restructures the IASR battery property table into a more usable format. + + The output table will have columns "storage_name" and the battery property names as + columns, with the values of those properties as the values in the table (converted to + numeric values where possible). Rows match storage names/mappings in the summary tables. + + Args: + battery_property_table: pd.DataFrame, `battery_properties` table from the IASR workbook. + + Returns: + pd.DataFrame, restructured battery property table. + """ + battery_properties = battery_property_table.set_index("Property").drop( + columns="Units" + ) + battery_properties = battery_properties.T.reset_index(names="storage_name") + + battery_properties.columns.name = None + + columns_to_make_numeric = [ + col for col in battery_properties.columns if col != "storage_name" + ] + for col in columns_to_make_numeric: + battery_properties[col] = pd.to_numeric( + battery_properties[col], errors="coerce" + ) + + return battery_properties + + def _merge_and_set_ecaa_battery_static_properties( df: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame] ): @@ -298,6 +315,16 @@ def _process_and_merge_opex( the column "NSW Low" or the relevant table, all other values are calculated from this base value multiplied by the O&M locational cost factor. This function merges in the post-LCF calculated values provided in the IASR workbook. + + Args: + df: New entrant storage template with opex column(s) filled in + table_data: OPEX table data to merge in + col_name: Name of the column to merge in, one of `'fom_$/kw/annum'` or `'vom_$/mwh_sent_out'` + table_attrs: Dictionary of attributes for the table to merge in + + Returns: + tuple[pd.DataFrame, str]: Updated dataframe with merged in values and `col_name` + returned for use in `_merge_table_data`. """ # update the mapping in this column to include storage name and the cost region initially given df[col_name] = df["storage_name"] + " " + df[col_name] @@ -307,6 +334,10 @@ def _process_and_merge_opex( for col in table_data.columns } ) + # standardise capitalisation of "storage" first: + table_data[table_attrs["table_lookup"]] = _standardise_storage_capitalisation( + table_data[table_attrs["table_lookup"]] + ) opex_table = table_data.melt( id_vars=[table_attrs["table_lookup"]], var_name="cost_region", @@ -319,15 +350,6 @@ def _process_and_merge_opex( opex_replacement_dict = ( opex_table[["mapping", "opex_value"]].set_index("mapping").squeeze().to_dict() ) - # use fuzzy matching in case of slight differences in storage names: - where_str = df[col_name].apply(lambda x: isinstance(x, str)) - df.loc[where_str, col_name] = _fuzzy_match_names( - df.loc[where_str, col_name], - opex_replacement_dict.keys(), - f"merging in the new entrant storage static property {col_name}", - not_match="existing", - threshold=90, - ) df[col_name] = df[col_name].replace(opex_replacement_dict) return df, col_name @@ -434,7 +456,7 @@ def _process_and_merge_connection_cost( battery_connection_costs_mapping = ( battery_connection_costs[["region_technology_mapping", "connection_cost_$/mw"]] .set_index("region_technology_mapping") - .squeeze() + .loc[:, "connection_cost_$/mw"] .to_dict() ) @@ -469,10 +491,16 @@ def _calculate_and_merge_tech_specific_lcfs( for each new entrant storage unit and merges into summary mapping table. """ # loads in the three tables needed - breakdown_ratios = iasr_tables["technology_cost_breakdown_ratios"].reset_index() + + # pre-process technology_cost_breakdown_ratios a little - there's something going on + # different between test and 'real' version of the index of this table so handle cases explicitly for now: + breakdown_ratios = iasr_tables["technology_cost_breakdown_ratios"] + drop_index = not (breakdown_ratios.index.name == "Technology") + breakdown_ratios = breakdown_ratios.reset_index(drop=drop_index) breakdown_ratios = breakdown_ratios.loc[ _where_any_substring_appears(breakdown_ratios["Technology"], ["battery"]) ].copy() + technology_specific_lcfs = iasr_tables["technology_specific_lcfs"] # loads all cols unless the str "O&M" is in col name locational_cost_factors = iasr_tables["locational_cost_factors"] @@ -499,15 +527,15 @@ def _calculate_and_merge_tech_specific_lcfs( columns={"Cost zones / Sub-region": "Location"}, inplace=True ) # ensures storage names in LCF tables match those in the summary table - for df_to_match_gen_names in [technology_specific_lcfs, breakdown_ratios]: - df_to_match_gen_names["Technology"] = _fuzzy_match_names( - df_to_match_gen_names["Technology"], + for df_to_match_batt_names in [technology_specific_lcfs, breakdown_ratios]: + df_to_match_batt_names["Technology"] = _fuzzy_match_names( + df_to_match_batt_names["Technology"], df["storage_name"].unique(), "calculating and merging in LCFs to static new entrant storage summary", not_match="existing", threshold=90, ) - df_to_match_gen_names.set_index("Technology", inplace=True) + df_to_match_batt_names.set_index("Technology", inplace=True) # use fuzzy matching to ensure that col names in tables to combine match up: fuzzy_column_renaming = _one_to_one_priority_based_fuzzy_matching( set(locational_cost_factors.columns.to_list()), @@ -558,6 +586,8 @@ def _safe_calculate_storage_duration_hours(row): ) return pd.NA elif row["maximum_capacity_mw"] == 0: + # we could also drop rows with maximum_capacity_mw == 0 altogether, but + # for the sake of keeping information around setting to 0.0 for now? return 0 else: return row["energy_capacity_mwh"] / row["maximum_capacity_mw"] @@ -578,37 +608,6 @@ def _safe_calculate_storage_duration_hours(row): return df -def _restructure_battery_property_table( - battery_property_table: pd.DataFrame, -) -> pd.DataFrame: - """Restructures the IASR battery property table into a more usable format. - - The output table will have columns "storage_name" and the battery property names as - columns, with the values of those properties as the values in the table (converted to - numeric values where possible). Rows match storage names/mappings in the summary tables. - - Args: - battery_property_table: pd.DataFrame, `battery_properties` table from the IASR workbook. - - Returns: - pd.DataFrame, restructured battery property table. - """ - battery_properties = battery_property_table.set_index("Property") - battery_properties = battery_properties.T.reset_index(names="storage_name") - - battery_properties.columns.name = None - - columns_to_make_numeric = [ - col for col in battery_properties.columns if col != "storage_name" - ] - for col in columns_to_make_numeric: - battery_properties[col] = pd.to_numeric( - battery_properties[col], errors="coerce" - ) - - return battery_properties - - def _add_and_clean_rez_ids( df: pd.DataFrame, rez_id_col_name: str, renewable_energy_zones: pd.DataFrame ) -> pd.DataFrame: diff --git a/src/ispypsa/translator/create_pypsa_friendly.py b/src/ispypsa/translator/create_pypsa_friendly.py index 436a7ff8..285fe7b1 100644 --- a/src/ispypsa/translator/create_pypsa_friendly.py +++ b/src/ispypsa/translator/create_pypsa_friendly.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Literal @@ -152,8 +153,10 @@ def create_pypsa_friendly_inputs( ignore_index=True, ) else: - # TODO: Log, improve error message - raise ValueError("No battery data returned from translator.") + logging.warning( + "No battery data returned from translator - no batteries added to model." + ) + # raise an error? Improve the message for sure buses = [] links = [] diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index b3d3e448..8b27cda1 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -95,22 +95,22 @@ "charging_efficiency_%": "efficiency_store", "discharging_efficiency_%": "efficiency_dispatch", # attributes used to filter/apply custom constraints: - "isp_resource_type": "isp_resource_type", "rez_id": "isp_rez_id", - # keeping technology_type because it's not defined anywhere else for the ECAA generators and could be useful for plotting/labelling? - "technology_type": "isp_technology_type", + "isp_resource_type": "isp_resource_type", } _BATTERY_ATTRIBUTE_ORDER = [ "name", "bus", "p_nom", - "max_hours", "p_nom_extendable", "carrier", + "max_hours", + "capital_cost", "build_year", "lifetime", - "capital_cost", + "efficiency_store", + "efficiency_dispatch", "isp_resource_type", "isp_rez_id", ] diff --git a/src/ispypsa/translator/storage.py b/src/ispypsa/translator/storage.py index 36dc4eb6..4d2399b4 100644 --- a/src/ispypsa/translator/storage.py +++ b/src/ispypsa/translator/storage.py @@ -1,10 +1,7 @@ -import re -from pathlib import Path -from typing import List, Literal +import logging import numpy as np import pandas as pd -from isp_trace_parser import get_data from ispypsa.translator.helpers import ( _add_investment_periods_as_build_years, @@ -48,8 +45,9 @@ def _translate_ecaa_batteries( ecaa_batteries = ispypsa_tables["ecaa_batteries"] if ecaa_batteries.empty: - # TODO: log - # raise error? + logging.warning( + "Templated table 'ecaa_batteries' is empty - no ECAA batteries will be included in this model." + ) return pd.DataFrame() # calculate lifetime based on expected closure_year - build_year: @@ -125,10 +123,11 @@ def _translate_new_entrant_batteries( `pd.DataFrame`: `PyPSA` style new entrant battery attributes in tabular format. """ - new_entrant_batteries = ispypsa_tables["new_entrant_batteries"] + new_entrant_batteries = ispypsa_tables["new_entrant_batteries"].copy() if new_entrant_batteries.empty: - # TODO: log - # raise error? + logging.warning( + "Templated table 'new_entrant_batteries' is empty - no new entrant batteries will be included in this model." + ) return pd.DataFrame() battery_attributes = _NEW_ENTRANT_BATTERY_ATTRIBUTES.copy() @@ -154,6 +153,13 @@ def _translate_new_entrant_batteries( new_entrant_batteries_all_build_years = _add_investment_periods_as_build_years( new_entrant_batteries, investment_periods ) + # then add build_year to battery name to maintain unique battery ID column + new_entrant_batteries_all_build_years["storage_name"] = ( + new_entrant_batteries_all_build_years["storage_name"].astype(str) + + "_" + + new_entrant_batteries_all_build_years["build_year"].astype(str) + ) + battery_df_with_build_costs = _add_new_entrant_battery_build_costs( new_entrant_batteries_all_build_years, ispypsa_tables["new_entrant_build_costs"], @@ -169,13 +175,6 @@ def _translate_new_entrant_batteries( ~battery_df_with_capital_costs["capital_cost"].isna() ].copy() - # then add build_year to battery name to maintain unique battery ID column - battery_df_with_capital_costs["storage_name"] = ( - battery_df_with_capital_costs["storage_name"] - + "_" - + battery_df_with_capital_costs["build_year"].astype(str) - ) - # add a p_nom column set to 0.0 for all new entrants: battery_df_with_capital_costs["p_nom"] = 0.0 @@ -236,6 +235,7 @@ def _add_new_entrant_battery_build_costs( _get_financial_year_int_from_string, args=("new entrant battery build costs", "fy"), ) + # make sure new_entrant_batteries has build_year column with ints: new_entrant_batteries["build_year"] = new_entrant_batteries["build_year"].astype( "int64" @@ -247,7 +247,8 @@ def _add_new_entrant_battery_build_costs( undefined_build_cost_batteries = ( new_entrants_with_build_costs[ new_entrants_with_build_costs["build_cost_$/mw"].isna() - ]["technology_type"] + ][["storage_name", "build_year"]] + .apply(tuple, axis=1) .unique() .tolist() ) diff --git a/tests/conftest.py b/tests/conftest.py index 527a31d4..48510fe5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -157,11 +157,13 @@ def sample_ispypsa_tables(csv_str_to_df): # Additional tables needed for new entrant generators: new_entrant_build_costs_csv = """ - technology, 2023_24_$/mw, 2024_25_$/mw, 2025_26_$/mw, 2026_27_$/mw, 2027_28_$/mw, 2028_29_$/mw - CCGT, 1900000, 1850000, 1800000, 1750000, 1700000, 1650000 - OCGT__(small__GT), 1600000, 1700000, 1650000, 1700000, 1750000, 1800000 - Large__scale__Solar__PV, 1700000, 1600000, 1500000, 1400000, 1300000, 1200000 - Wind, 2900000, 2800000, 2700000, 2600000, 2500000, 2400000 + technology, 2023_24_$/mw, 2024_25_$/mw, 2025_26_$/mw, 2026_27_$/mw, 2027_28_$/mw, 2028_29_$/mw + CCGT, 1900000, 1850000, 1800000, 1750000, 1700000, 1650000 + OCGT__(small__GT), 1600000, 1700000, 1650000, 1700000, 1750000, 1800000 + Large__scale__Solar__PV, 1700000, 1600000, 1500000, 1400000, 1300000, 1200000 + Wind, 2900000, 2800000, 2700000, 2600000, 2500000, 2400000 + Battery__Storage__(2hrs__storage), 3000000, 2900000, 2800000, 2700000, 2600000, 2500000 + Battery__Storage__(4hrs__storage), 4000000, 3900000, 3800000, 3700000, 3600000, 3500000 """ tables["new_entrant_build_costs"] = csv_str_to_df(new_entrant_build_costs_csv) @@ -250,6 +252,22 @@ def sample_ispypsa_tables(csv_str_to_df): gpg_emissions_reduction_biomethane_csv ) + # ECAA battery table: + ecaa_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + """ + tables["ecaa_batteries"] = csv_str_to_df(ecaa_batteries_csv) + + # New entrant battery table: + new_entrant_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + """ + tables["new_entrant_batteries"] = csv_str_to_df(new_entrant_batteries_csv) + return tables diff --git a/tests/test_model/test_add_generators.py b/tests/test_model/test_add_generators.py index ef3d8dd9..c591c0f8 100644 --- a/tests/test_model/test_add_generators.py +++ b/tests/test_model/test_add_generators.py @@ -1,14 +1,9 @@ -from pathlib import Path - import pandas as pd import pypsa import pytest -from pandas.testing import assert_series_equal from ispypsa.pypsa_build.generators import ( _add_generator_to_network, - _get_marginal_cost_timeseries, - _get_trace_data, ) diff --git a/tests/test_model/test_add_storage.py b/tests/test_model/test_add_storage.py new file mode 100644 index 00000000..4e9d5f0a --- /dev/null +++ b/tests/test_model/test_add_storage.py @@ -0,0 +1,109 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.model.storage import ( + _add_batteries_to_network, + _add_battery_to_network, +) + + +@pytest.fixture +def mock_network(csv_str_to_df): + """Create a minimal PyPSA network for testing.""" + network = pypsa.Network() + network.add("Bus", "test_bus") + + # Create sample trace data + snapshots_csv = """ + investment_periods, snapshots + 2024, 2023-07-01__12:00:00 + 2024, 2024-04-01__12:00:00 + 2025, 2024-07-01__12:00:00 + 2025, 2025-04-01__12:00:00 + """ + snapshots = csv_str_to_df(snapshots_csv) + snapshots["snapshots"] = pd.to_datetime(snapshots["snapshots"]) + + # set network snapshots: + network.snapshots = snapshots + + return network + + +def test_add_battery_to_network(mock_network): + sample_battery = dict( + name="test_battery", + bus="test_bus", + p_nom=100.0, + p_nom_extendable=False, + carrier="Battery", + max_hours=2, + capital_cost=0.0, + build_year=2015, + lifetime=20, + efficiency_store=0.92, + efficiency_dispatch=0.92, + isp_resource_type="Battery Storage 2h", + isp_rez_id="", + ) + + _add_battery_to_network(mock_network, sample_battery) + + # Check the battery was added correctly + assert "test_battery" in mock_network.storage_units.index + assert "isp_resource_type" not in mock_network.storage_units.columns + assert mock_network.storage_units.at["test_battery", "max_hours"] == 2 + + # Check that some key default values have been set: + assert mock_network.storage_units.at["test_battery", "p_max_pu"] == 1 + assert mock_network.storage_units.at["test_battery", "p_min_pu"] == -1 + + +def test_add_batteries_to_network(mock_network, csv_str_to_df): + sample_batteries_csv = """ + name, bus, p_nom, p_nom_extendable, carrier, max_hours, capital_cost, build_year, lifetime, efficiency_store, efficiency_dispatch, isp_resource_type, isp_rez_id + test_battery_1, test_bus, 100.0, False, Battery, 2, 0.0, 2015, 20, 0.92, 0.92, Battery__Storage__2h, rez_A + test_battery_2, test_bus, 100.0, False, Battery, 4, 0.0, 2015, 20, 0.91, 0.91, Battery__Storage__4h, rez_B + new_entrant_1, test_bus, 0.0, True, Battery, 1, 100000.0, 2015, 20, 0.92, 0.92, Battery__Storage__1h, + new_entrant_2, test_bus, 0.0, True, Battery, 8, 400000.0, 2015, 20, 0.93, 0.93, Battery__Storage__8h, + """ + sample_batteries = csv_str_to_df(sample_batteries_csv) + + _add_batteries_to_network(mock_network, sample_batteries) + + # Check the batteries were all added correctly - check each feature: + assert set(mock_network.storage_units.index) == set( + [ + "test_battery_1", + "test_battery_2", + "new_entrant_1", + "new_entrant_2", + ] + ) + assert (mock_network.storage_units.bus == "test_bus").all() + assert (mock_network.storage_units.carrier == "Battery").all() + + characteristics_that_should_match_exactly = [ + "bus", + "p_nom", + "p_nom_extendable", + "carrier", + "max_hours", + "capital_cost", + "build_year", + "lifetime", + "efficiency_store", + "efficiency_dispatch", + ] + sample_batteries = sample_batteries.set_index("name") + for test_battery in sample_batteries.index: + for characteristic in characteristics_that_should_match_exactly: + assert ( + mock_network.storage_units.at[test_battery, characteristic] + == sample_batteries.at[test_battery, characteristic] + ) + + # Check that some key default values have been set: + assert (mock_network.storage_units.p_max_pu == 1).all() + assert (mock_network.storage_units.p_min_pu == -1).all() diff --git a/tests/test_model/test_pypsa_friendly_inputs/test_custom_constraints/batteries.csv b/tests/test_model/test_pypsa_friendly_inputs/test_custom_constraints/batteries.csv new file mode 100644 index 00000000..1f7dfa44 --- /dev/null +++ b/tests/test_model/test_pypsa_friendly_inputs/test_custom_constraints/batteries.csv @@ -0,0 +1,3 @@ +name,bus,p_nom,p_nom_extendable,carrier,max_hours,capital_cost,build_year,lifetime,efficiency_store,efficiency_dispatch,isp_resource_type,isp_rez_id +Wandoan South BESS,Q8,100.0,False,Battery,1.5,0.0,2025,26.0,91.6515138991168,91.6515138991168,Battery Storage 2h,Q8 +battery_storage_1h_gg_2025,GG,0.0,True,Battery,1.0,91564.95598203539,2025,20.0,91.6515138991168,91.6515138991168,Battery Storage 1h, diff --git a/tests/test_templater/test_storage.py b/tests/test_templater/test_storage.py new file mode 100644 index 00000000..99dc0417 --- /dev/null +++ b/tests/test_templater/test_storage.py @@ -0,0 +1,562 @@ +import logging +from pathlib import Path + +import pandas as pd +import pytest + +from ispypsa.data_fetch import read_csvs +from ispypsa.templater.lists import _MINIMUM_REQUIRED_BATTERY_COLUMNS +from ispypsa.templater.mappings import ( + _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP, + _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP, +) +from ispypsa.templater.storage import ( + _add_and_clean_rez_ids, + _add_closure_year_column, + _add_isp_resource_type_column, + _calculate_and_merge_tech_specific_lcfs, + _calculate_storage_duration_hours, + _process_and_merge_connection_cost, + _process_and_merge_opex, + _restructure_battery_property_table, + _template_battery_properties, +) + + +def test_battery_templater(workbook_table_cache_test_path: Path): + iasr_tables = read_csvs(workbook_table_cache_test_path) + ecaa_batteries, new_entrant_batteries = _template_battery_properties(iasr_tables) + + for ecaa_property_col in _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP.keys(): + if ( + ecaa_property_col in _MINIMUM_REQUIRED_BATTERY_COLUMNS + and "date" not in ecaa_property_col + ): + assert all( + ecaa_batteries[ecaa_property_col].apply( + lambda x: True if not isinstance(x, str) else False + ) + ) + + for ( + new_entrant_property_col + ) in _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP.keys(): + if ( + new_entrant_property_col in _MINIMUM_REQUIRED_BATTERY_COLUMNS + and "date" not in new_entrant_property_col + ): + assert all( + new_entrant_batteries[new_entrant_property_col].apply( + lambda x: True if not isinstance(x, str) else False + ) + ) + + # limited test CSV contains only "Existing" ECAA battery + assert all(ecaa_batteries["status"] == "Existing") + assert all(new_entrant_batteries["status"] == "New Entrant") + + all_columns = ( + ecaa_batteries.columns.tolist() + new_entrant_batteries.columns.tolist() + ) + for column in all_columns: + assert column in _MINIMUM_REQUIRED_BATTERY_COLUMNS + + +def test_add_closure_year_column(csv_str_to_df): + """Test the _add_closure_year_column function with various scenarios.""" + # Setup test data + ecaa_batteries_csv = """ + storage_name, technology_type, region_id, closure_year + Existing__Battery, Battery__Storage, NSW, Existing__Battery + New_Battery_No_Closure, Battery__Storage, QLD, New_Battery_No_Closure + """ + ecaa_batteries = csv_str_to_df(ecaa_batteries_csv) + + closure_years_csv = """ + Generator__name, DUID, Expected__Closure__Year__(Calendar__year) + Existing__Battery, EB01, 2035 + Existing__Battery, EB02, 2036 + """ + closure_years = csv_str_to_df(closure_years_csv) + + # Execute function + result = _add_closure_year_column(ecaa_batteries, closure_years) + + # Expected result + expected_csv = """ + storage_name, technology_type, region_id, closure_year + Existing__Battery, Battery__Storage, NSW, 2035 + New_Battery_No_Closure, Battery__Storage, QLD, -1 + """ + expected = csv_str_to_df(expected_csv) + expected = expected.fillna(pd.NA) + + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + ) + + +def test_add_closure_year_column_empty_ecaa_df(csv_str_to_df): + """Test edge cases for the _add_closure_year_column function.""" + + # Setup test data + ecaa_batteries_csv = """ + storage_name, technology_type, region_id, closure_year + Battery_A, Battery__Storage, NSW, Battery_A + Battery_B, Battery__Storage, SA, Battery_B + """ + ecaa_batteries = csv_str_to_df(ecaa_batteries_csv) + + # Case 1a: Empty closure years dataframe + empty_closure = pd.DataFrame( + columns=["Generator name", "DUID", "Expected Closure Year (Calendar year)"] + ) + + # Execute function + result = _add_closure_year_column(ecaa_batteries, empty_closure) + + # Expected result + expected_csv = """ + storage_name, technology_type, region_id, closure_year + Battery_A, Battery__Storage, NSW, -1 + Battery_B, Battery__Storage, SA, -1 + """ + expected = csv_str_to_df(expected_csv) + expected = expected.fillna(pd.NA) + + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + ) + + +def test_restructure_battery_property_table( + csv_str_to_df, workbook_table_cache_test_path: Path +): + """Test the _restructure_battery_property_table function.""" + + # grab test table: + iasr_tables = read_csvs(workbook_table_cache_test_path) + battery_properties = iasr_tables["battery_properties"] + + # Execute function + result = _restructure_battery_property_table(battery_properties) + + # Expected result - restructured with batteries as rows + expected_csv = """ + storage_name, Maximum__power, Energy__capacity, Charge__efficiency__(utility), Discharge__efficiency__(utility), Round__trip__efficiency__(utility), Annual__degradation__(utility) + Battery__storage__(1hr__storage), 1.0, 1.0, 91.7, 91.7, 84.0, 1.8 + Battery__storage__(2hrs__storage), 1.0, 2.0, 91.7, 91.7, 84.0, 1.8 + Battery__storage__(4hrs__storage), 1.0, 4.0, 92.2, 92.2, 85.0, 1.8 + Battery__storage__(8hrs__storage), 1.0, 8.0, 91.1, 91.1, 83.0, 1.8 + """ + expected = csv_str_to_df(expected_csv) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + ) + + +def test_calculate_storage_duration_hours_edge_cases(csv_str_to_df, caplog): + """Test edge cases for the _calculate_storage_duration_hours function.""" + # Case 1: Empty DataFrame + empty_df = pd.DataFrame( + columns=["storage_name", "maximum_capacity_mw", "energy_capacity_mwh"] + ) + result_empty = _calculate_storage_duration_hours(empty_df) + assert result_empty.empty, "Empty DataFrame should return empty DataFrame" + + # Case 2: Missing required columns + missing_columns_df = csv_str_to_df(""" + storage_name, some_other_column + Battery_A, value + """) + with pytest.raises(KeyError): + _calculate_storage_duration_hours(missing_columns_df) + + # Case 3: All rows have missing values + all_missing_df = csv_str_to_df(""" + storage_name, maximum_capacity_mw, energy_capacity_mwh + Battery_A, , + Battery_B, , 100 + Battery_C, 100, + """) + + # Capture logs to verify warnings are issued + with caplog.at_level(logging.WARNING): + result_all_missing = _calculate_storage_duration_hours(all_missing_df) + + # Check that warnings were logged for each row + assert "Battery_A" in caplog.text + assert "Battery_B" in caplog.text + assert "Battery_C" in caplog.text + assert "missing maximum_capacity_mw or energy_capacity_mwh value" in caplog.text + + # Result should be empty as all rows had missing values (missing values get dropped) + assert result_all_missing.empty + + # Case 4: Non-numeric values in numeric columns + non_numeric_df = csv_str_to_df(""" + storage_name, maximum_capacity_mw, energy_capacity_mwh + Battery_A, text, 400 + Battery_B, 100, text + """) + + # Clear previous logs + caplog.clear() + + with caplog.at_level(logging.WARNING): + result_non_numeric = _calculate_storage_duration_hours(non_numeric_df) + + # Check that warnings were logged + assert "Battery_A" in caplog.text + assert "Battery_B" in caplog.text + + # Result should be empty as all rows had invalid values + assert result_non_numeric.empty + + # Case 5: Mixed valid and invalid rows + mixed_df = csv_str_to_df(""" + storage_name, maximum_capacity_mw, energy_capacity_mwh + Battery_Valid, 100, 400 + Battery_Invalid, , 100 + Battery_Zero, 0, 200 + """) + + result_mixed = _calculate_storage_duration_hours(mixed_df) + + # Expected result - only valid rows and zero capacity row should remain + expected_csv = """ + storage_name, maximum_capacity_mw, energy_capacity_mwh, storage_duration_hours + Battery_Valid, 100, 400, 4.0 + Battery_Zero, 0, 200, 0.0 + """ + expected = csv_str_to_df(expected_csv) + + # Assert results match expected + pd.testing.assert_frame_equal( + result_mixed.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + ) + + +def test_add_isp_resource_type_column(csv_str_to_df): + """Test the _add_isp_resource_type_column function.""" + # Setup test data - include different capitalisations too + storage_df_csv = """ + storage_name, isp_resource_type + Battery_A, All__Battery__storage__(2hrs__storage) + Battery_B, All__Battery__Storage__(1hr__storage) + Non_Battery, Not__Matching__String__Pattern + """ + storage_df = csv_str_to_df(storage_df_csv) + + # Execute function + result = _add_isp_resource_type_column(storage_df) + + # Expected result - should extract duration and format consistently + expected_csv = """ + storage_name, isp_resource_type + Battery_A, Battery__Storage__2h + Battery_B, Battery__Storage__1h + Non_Battery, None + """ + expected = csv_str_to_df(expected_csv) + + # Make nan values from result and expected the same type of nan: + result = result.fillna(pd.NA) + expected = expected.fillna(pd.NA) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + ) + + +def test_add_and_clean_rez_ids(csv_str_to_df): + """Test the _add_and_clean_rez_ids function.""" + # Setup test data + storage_df_csv = """ + storage_name, region_id, sub_region_id, rez_location + Battery_NSW, NSW, NNSW, North__West__NSW + Battery_QLD, QLD, SQ, Darling__Downs + Battery_Non_REZ, NSW, NNSW, + """ + storage_df = csv_str_to_df(storage_df_csv) + + rez_df_csv = """ + ID, Name, NEM__Region, NTNDP__Zone, ISP__Sub-region, Regional__Cost__Zones + N1, North__West__NSW, NSW, NNS, NNSW, Medium + Q8, Darling__Downs, QLD, SWQ, SQ, Low + N2, New__England, NSW, NNS, NNSW, Low + """ + rez_df = csv_str_to_df(rez_df_csv) + + # Execute function + result = _add_and_clean_rez_ids(storage_df, "rez_id", rez_df) + result = result.fillna(pd.NA) + + # Expected result - should add rez_id column + expected_csv = """ + storage_name, region_id, sub_region_id, rez_location, rez_id + Battery_NSW, NSW, NNSW, N1, N1 + Battery_QLD, QLD, SQ, Q8, Q8 + Battery_Non_REZ, NSW, NNSW, + """ + expected = csv_str_to_df(expected_csv) + expected = expected.fillna(pd.NA) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + ) + + +def test_add_and_clean_rez_ids_special_names(csv_str_to_df): + """Test _add_and_clean_rez_ids with special REZ name mappings.""" + # Setup test data with special REZ names that should be standardized + storage_df_csv = """ + storage_name, region_id, sub_region_id, rez_location + Battery_TAS_NE, TAS, TAS, North__East__Tasmania__Coast + Battery_TAS_NW, TAS, TAS, North__West__Tasmania__Coast + Battery_VIC, VIC, VIC, Portland__Coast + """ + storage_df = csv_str_to_df(storage_df_csv) + + rez_df_csv = """ + ID, Name, NEM__Region + T1, North__Tasmania__Coast, TAS + V2, Southern__Ocean, VIC + """ + rez_df = csv_str_to_df(rez_df_csv) + + # Execute function + result = _add_and_clean_rez_ids(storage_df, "rez_id", rez_df) + result = result.fillna(pd.NA) + + # Expected result - should standardize names and map to correct REZ IDs for all columns + # with "rez" or "region_id" in the name (where REZ names are present) + expected_csv = """ + storage_name, region_id, sub_region_id, rez_location, rez_id + Battery_TAS_NE, TAS, TAS, T1, T1 + Battery_TAS_NW, TAS, TAS, T1, T1 + Battery_VIC, VIC, VIC, V2, V2 + """ + expected = csv_str_to_df(expected_csv) + expected = expected.fillna(pd.NA) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + ) + + +def test_process_and_merge_opex(csv_str_to_df, workbook_table_cache_test_path: Path): + """Test the _process_and_merge_opex function with various scenarios.""" + iasr_tables = read_csvs(workbook_table_cache_test_path) + new_entrant_storage_df = csv_str_to_df(""" + storage_name, technology_type, region_id, sub_region_id, fom_$/kw/annum + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), NSW, NNSW, NSW__Low + Battery__Storage__(1hr__storage), Battery__Storage__(1hr__storage), QLD, SQ, QLD__Low + """) + + new_entrant_fixed_opex = iasr_tables["fixed_opex_new_entrants"] + + table_attrs = dict( + table="fixed_opex_new_entrants", + table_lookup="Generator", + table_col_prefix="Fixed OPEX ($/kW sent out/year)", + ) + + # Execute function for fixed O&M costs + result_df, result_col = _process_and_merge_opex( + new_entrant_storage_df.copy(), + new_entrant_fixed_opex, + "fom_$/kw/annum", + table_attrs, + ) + + # Expected result for fixed O&M costs + expected_fixed_csv = """ + storage_name, technology_type, region_id, sub_region_id, fom_$/kw/annum + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), NSW, NNSW, 10.799929999999998 + Battery__Storage__(1hr__storage), Battery__Storage__(1hr__storage), QLD, SQ, 7.592029999999999 + """ + expected_fixed = csv_str_to_df(expected_fixed_csv) + + # Assert fixed O&M results match expected + pd.testing.assert_frame_equal( + result_df.sort_values("storage_name").reset_index(drop=True), + expected_fixed.sort_values("storage_name").reset_index(drop=True), + check_exact=False, + check_dtype=False, + ) + assert result_col == "fom_$/kw/annum" + + # Test edge case: Empty storage dataframe + empty_df = pd.DataFrame( + columns=["storage_name", "technology_type", "fom_$/kw/annum"] + ) + result_empty_df, result_empty_col = _process_and_merge_opex( + empty_df, new_entrant_fixed_opex, "fom_$/kw/annum", table_attrs + ) + assert result_empty_df.empty + assert result_empty_col == "fom_$/kw/annum" + + # Test edge case: Empty opex table + empty_opex = pd.DataFrame( + columns=["Generator", "Fixed OPEX ($/kW sent out/year)_NSW Low"] + ) + result_empty_opex_df, result_empty_opex_col = _process_and_merge_opex( + new_entrant_storage_df.copy(), empty_opex, "fom_$/kw/annum", table_attrs + ) + # Should return original dataframe with values unchanged + assert "fom_$/kw/annum" in result_empty_opex_df.columns + assert result_empty_opex_col == "fom_$/kw/annum" + + +def test_calculate_and_merge_tech_specific_lcfs( + csv_str_to_df, workbook_table_cache_test_path: Path +): + """Test the _calculate_and_merge_tech_specific_lcfs function.""" + + iasr_tables = read_csvs(workbook_table_cache_test_path) + + # Setup test data + storage_df_csv = """ + storage_name, technology_type, technology_specific_lcf_% + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), NSW__Low + Battery__Storage__(1hr__storage), Battery__Storage__(1hr__storage), QLD__Low + """ + storage_df = csv_str_to_df(storage_df_csv) + + # Execute function + result = _calculate_and_merge_tech_specific_lcfs( + storage_df, iasr_tables, "technology_specific_lcf_%" + ) + + # Expected result + expected_csv = """ + storage_name, technology_type, technology_specific_lcf_% + Battery__Storage__(1hr__storage), Battery__Storage__(1hr__storage), 100.0 + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), 100.0 + """ + expected = csv_str_to_df(expected_csv) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + check_exact=False, # allow for floating point errors + rtol=1e-2, + ) + + # Test edge case: Empty storage dataframe + empty_df = pd.DataFrame( + columns=["storage_name", "technology_type", "technology_specific_lcf_%"] + ) + result_empty = _calculate_and_merge_tech_specific_lcfs( + empty_df, iasr_tables, "technology_specific_lcf_%" + ) + assert result_empty.empty + assert "technology_specific_lcf_%" in result_empty.columns + + # Test edge case: Missing location mapping + missing_location_csv = """ + storage_name, technology_type, technology_specific_lcf_% + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), Nonexistant__Region + """ + missing_location_df = csv_str_to_df(missing_location_csv) + + result_missing = _calculate_and_merge_tech_specific_lcfs( + missing_location_df, iasr_tables, "technology_specific_lcf_%" + ) + + # Expected result for missing location - should use default value of 1.0 + expected_missing_csv = """ + storage_name, technology_type, technology_specific_lcf_% + Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), + """ + expected_missing = csv_str_to_df(expected_missing_csv) + # match nan types: + expected_missing = expected_missing.fillna(pd.NA) + result_missing = result_missing.fillna(pd.NA) + + pd.testing.assert_frame_equal( + result_missing.sort_values("storage_name").reset_index(drop=True), + expected_missing.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + ) + + +def test_process_and_merge_connection_cost(csv_str_to_df): + """Test the _process_and_merge_connection_cost function.""" + # Setup test data + storage_df_csv = """ + storage_name, connection_cost_rez/_region_id, connection_cost_technology + Battery__Storage__(2hrs__storage), NSW, 2hr__Battery__Storage + Battery__Storage__(1hr__storage), QLD, 1hr__Battery__Storage + """ + storage_df = csv_str_to_df(storage_df_csv) + + connection_costs_csv = """ + Region, 1__hr__Battery__Storage, 2__hr__Battery__Storage + NSW, 50, 55 + QLD, 45, 48 + """ + connection_costs_table = csv_str_to_df(connection_costs_csv) + + # Execute function + result = _process_and_merge_connection_cost(storage_df, connection_costs_table) + + # Expected result - connection costs should be in $/MW (multiplied by 1000) + expected_csv = """ + storage_name, connection_cost_rez/_region_id, connection_cost_technology, connection_cost_$/mw + Battery__Storage__(2hrs__storage), NSW, 2hr__Battery__Storage, 55000.0 + Battery__Storage__(1hr__storage), QLD, 1hr__Battery__Storage, 45000.0 + """ + expected = csv_str_to_df(expected_csv) + + # Assert results match expected + pd.testing.assert_frame_equal( + result.sort_values("storage_name").reset_index(drop=True), + expected.sort_values("storage_name").reset_index(drop=True), + check_dtype=False, + ) + + # Test edge case: Empty storage dataframe + empty_df = pd.DataFrame( + columns=[ + "storage_name", + "technology_type", + "connection_cost_rez/_region_id", + "connection_cost_technology", + ] + ) + result_empty = _process_and_merge_connection_cost(empty_df, connection_costs_table) + assert result_empty.empty + assert "connection_cost_$/mw" in result_empty.columns + + # Test edge case: Missing connection costs that should raise ValueError. Use the same storage_df to test. + # Create a connection costs table that will result in NaN values after merging + incomplete_costs_csv = """ + Region, 1__hr__Battery__Storage + QLD, 50 + """ + incomplete_costs = csv_str_to_df(incomplete_costs_csv) + + with pytest.raises( + ValueError, + match=r"Missing connection costs for the following batteries: \['Battery Storage \(2hrs storage\)'\]", + ): + _process_and_merge_connection_cost(storage_df, incomplete_costs) diff --git a/tests/test_translator/test_translate_storage.py b/tests/test_translator/test_translate_storage.py new file mode 100644 index 00000000..ee1aced7 --- /dev/null +++ b/tests/test_translator/test_translate_storage.py @@ -0,0 +1,355 @@ +import logging + +import pandas as pd +import pytest + +from ispypsa.translator.mappings import ( + _BATTERY_ATTRIBUTE_ORDER, + _ECAA_BATTERY_ATTRIBUTES, + _NEW_ENTRANT_BATTERY_ATTRIBUTES, +) +from ispypsa.translator.storage import ( + _add_new_entrant_battery_build_costs, + _calculate_annuitised_new_entrant_battery_capital_costs, + _translate_ecaa_batteries, + _translate_new_entrant_batteries, +) + + +def test_translate_ecaa_batteries_empty_input(caplog): + """Test that empty input returns empty output.""" + ispypsa_tables = {"ecaa_batteries": pd.DataFrame()} + investment_periods = [2025] + + with caplog.at_level(logging.WARNING): + result = _translate_ecaa_batteries(ispypsa_tables, investment_periods) + + assert ( + "Templated table 'ecaa_batteries' is empty - no ECAA batteries will be included in this model." + in caplog.text + ) + assert result.empty + + +def test_translate_ecaa_batteries_basic(csv_str_to_df): + """Test basic functionality of ECAA batteries translation.""" + # Create test input data + ecaa_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + """ + + ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} + investment_periods = [2025] + + result = _translate_ecaa_batteries(ispypsa_tables, investment_periods) + + # Assert expected columns and values + for templater_name, translated_name in _ECAA_BATTERY_ATTRIBUTES.items(): + assert translated_name in result.columns + + assert "bus" in result.columns # check this separately (not in mapping) + assert not result["p_nom_extendable"].any() # All should be False + assert (result["capital_cost"] == 0.0).all() # All should be 0.0 + + # Check that no extra columns were added: + extra_cols = set(result.columns) - set(_BATTERY_ATTRIBUTE_ORDER) + assert not extra_cols + + +def test_translate_ecaa_batteries_regional_granularity(csv_str_to_df): + """Test different regional granularity settings.""" + # Create test input data + ecaa_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + """ + + ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} + investment_periods = [2025] + + # Test sub_regions setting + result_sub = _translate_ecaa_batteries( + ispypsa_tables, investment_periods, regional_granularity="sub_regions" + ) + assert (result_sub["bus"] == "NNSW").all() + + # Test nem_regions setting + result_nem = _translate_ecaa_batteries( + ispypsa_tables, investment_periods, regional_granularity="nem_regions" + ) + assert (result_nem["bus"] == "NSW").all() + + # Test single_region setting + result_single = _translate_ecaa_batteries( + ispypsa_tables, investment_periods, regional_granularity="single_region" + ) + assert (result_single["bus"] == "NEM").all() + + +def test_translate_ecaa_batteries_rez_handling(csv_str_to_df): + """Test REZ handling options.""" + # Create test input with REZ + ecaa_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h + Battery2, CNSW, NSW, N3, 2022-07-01, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + """ + + ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} + investment_periods = [2025] + + # Test discrete_nodes + result_discrete = _translate_ecaa_batteries( + ispypsa_tables, investment_periods, rez_handling="discrete_nodes" + ) + assert set(result_discrete["bus"].values) == set(["N3", "CNSW"]) + + # Test attached_to_parent_node + result_attached = _translate_ecaa_batteries( + ispypsa_tables, investment_periods, rez_handling="attached_to_parent_node" + ) + assert (result_attached["bus"] == "CNSW").all() + assert "N3" not in result_attached["bus"].values + + +def test_translate_ecaa_batteries_lifetime_calculation(csv_str_to_df): + """Test lifetime calculation based on closure year.""" + # Create test input + ecaa_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h + Battery2, CNSW, NSW, , 2022-07-01, 2045, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + Battery3, CNSW, NSW, , 2010-04-01, 2020, 300, 0.92, 0.92, 8, Battery, Battery__Storage__8h + """ + + ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} + investment_periods = [2025] + + result = _translate_ecaa_batteries(ispypsa_tables, investment_periods) + + # Battery3 should be filtered out (closure_year < investment_period) + expected_result_csv = """ + name, bus, p_nom, p_nom_extendable, carrier, max_hours, capital_cost, build_year, lifetime, efficiency_store, efficiency_dispatch, isp_resource_type, isp_rez_id + Battery1, CNSW, 100.0, False, Battery, 4, 0.0, 2020, 15, 0.95, 0.95, Battery__Storage__4h, + Battery2, CNSW, 200.0, False, Battery, 2, 0.0, 2023, 20, 0.90, 0.90, Battery__Storage__2h, + """ + expected_result = csv_str_to_df(expected_result_csv) + + pd.testing.assert_frame_equal( + result.reset_index(drop=True).sort_values("name"), + expected_result.reset_index(drop=True).sort_values("name"), + check_dtype=False, + ) + + +def test_translate_new_entrant_batteries_empty_input(caplog): + """Test that empty input returns empty output.""" + ispypsa_tables = {"new_entrant_batteries": pd.DataFrame()} + investment_periods = [2025] + wacc = 0.05 + + with caplog.at_level(logging.WARNING): + result = _translate_new_entrant_batteries( + ispypsa_tables, investment_periods, wacc + ) + + assert ( + "Templated table 'new_entrant_batteries' is empty - no new entrant batteries will be included in this model." + in caplog.text + ) + assert result.empty + + +def test_translate_new_entrant_batteries_basic(csv_str_to_df, sample_ispypsa_tables): + """Test basic functionality of new entrant batteries translation.""" + # Create test input data + new_entrant_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + """ + + ispypsa_tables = { + "new_entrant_batteries": csv_str_to_df(new_entrant_batteries_csv), + "new_entrant_build_costs": sample_ispypsa_tables["new_entrant_build_costs"], + } + investment_periods = [2025, 2027] + wacc = 0.05 + + result = _translate_new_entrant_batteries(ispypsa_tables, investment_periods, wacc) + + # Assert expected columns and values + for templater_name, translated_name in _NEW_ENTRANT_BATTERY_ATTRIBUTES.items(): + assert translated_name in result.columns + + assert "bus" in result.columns # check this separately (it's not in mapping) + assert result["p_nom_extendable"].all() # All should be True + assert len(result) == 2 + + +def test_translate_new_entrant_batteries_regional_granularity( + csv_str_to_df, sample_ispypsa_tables +): + """Test different regional granularity settings for new entrant batteries.""" + new_entrant_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + """ + + ispypsa_tables = { + "new_entrant_batteries": csv_str_to_df(new_entrant_batteries_csv), + "new_entrant_build_costs": sample_ispypsa_tables["new_entrant_build_costs"], + } + investment_periods = [2025] + wacc = 0.05 + + # Test sub_regions setting + result_sub = _translate_new_entrant_batteries( + ispypsa_tables, investment_periods, wacc, regional_granularity="sub_regions" + ) + assert (result_sub["bus"] == "CNSW").all() + + # Test nem_regions setting + result_nem = _translate_new_entrant_batteries( + ispypsa_tables, investment_periods, wacc, regional_granularity="nem_regions" + ) + assert (result_nem["bus"] == "NSW").all() + + # Test single_region setting + result_single = _translate_new_entrant_batteries( + ispypsa_tables, investment_periods, wacc, regional_granularity="single_region" + ) + assert (result_single["bus"] == "NEM").all() + + +def test_translate_new_entrant_batteries_rez_handling( + csv_str_to_df, sample_ispypsa_tables +): + """Test REZ handling options for new entrant batteries.""" + new_entrant_batteries_csv = """ + storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, N3, Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + """ + + ispypsa_tables = { + "new_entrant_batteries": csv_str_to_df(new_entrant_batteries_csv), + "new_entrant_build_costs": sample_ispypsa_tables["new_entrant_build_costs"], + } + investment_periods = [2025] + wacc = 0.05 + + # Test discrete_nodes + result_discrete = _translate_new_entrant_batteries( + ispypsa_tables, investment_periods, wacc, rez_handling="discrete_nodes" + ) + assert set(result_discrete["bus"].values) == set(["N3", "CNSW"]) + + # Test attached_to_parent_node + result_attached = _translate_new_entrant_batteries( + ispypsa_tables, + investment_periods, + wacc, + regional_granularity="sub_regions", + rez_handling="attached_to_parent_node", + ) + assert (result_attached["bus"] == "CNSW").all() + assert "N3" not in result_attached["bus"].values + + +def test_add_new_entrant_build_costs(csv_str_to_df, sample_ispypsa_tables): + """Test that build costs are correctly merged into the batteries table.""" + batteries_csv = """ + storage_name, technology_type, build_year + Battery_4h, Battery__Storage__(4hrs__storage), 2025 + Battery_2h, Battery__Storage__(2hrs__storage), 2027 + """ + batteries_df = csv_str_to_df(batteries_csv) + + build_costs_df = sample_ispypsa_tables["new_entrant_build_costs"] + + # Call the function + result = _add_new_entrant_battery_build_costs(batteries_df, build_costs_df) + + expected_result_csv = """ + storage_name, technology_type, build_year, build_cost_$/mw + Battery_4h, Battery__Storage__(4hrs__storage), 2025, 3900000 + Battery_2h, Battery__Storage__(2hrs__storage), 2027, 2700000 + """ + expected_result = csv_str_to_df(expected_result_csv) + + pd.testing.assert_frame_equal( + result.reset_index(drop=True).sort_values("storage_name"), + expected_result.reset_index(drop=True).sort_values("storage_name"), + check_dtype=False, + ) + + +def test_add_new_entrant_build_costs_missing_build_year( + csv_str_to_df, sample_ispypsa_tables +): + """Test that the function raises an error when build_year column is missing.""" + batteries_csv = """ + storage_name, technology_type + Battery_4hr, Battery_4hr + """ + batteries_df = csv_str_to_df(batteries_csv) + + build_costs_df = sample_ispypsa_tables["new_entrant_build_costs"] + + with pytest.raises( + ValueError, + match="new_entrant_batteries table must have column 'build_year' to merge in build costs.", + ): + _add_new_entrant_battery_build_costs(batteries_df, build_costs_df) + + +def test_add_new_entrant_build_costs_undefined_build_costs( + csv_str_to_df, sample_ispypsa_tables +): + """Test that the function raises an error when build costs are undefined for some batteries.""" + batteries_csv = """ + storage_name, technology_type, build_year + Battery_4hr, Battery__Storage__(4hrs__storage), 2028 + Battery_2hr, Battery__Storage__(2hrs__storage), 2024 + NonExistentType, NonExistentType, 2025 + NonExistentYear, Battery__Storage__(2hrs__storage), 2010 + """ + batteries_df = csv_str_to_df(batteries_csv) + + build_costs_df = sample_ispypsa_tables["new_entrant_build_costs"] + with pytest.raises( + ValueError, + match=r"Undefined build costs for new entrant batteries: \[\('NonExistentType', 2025\), \('NonExistentYear', 2010\)\]", + ): + _add_new_entrant_battery_build_costs(batteries_df, build_costs_df) + + +def test_calculate_annuitised_new_entrant_battery_capital_costs(csv_str_to_df): + """Test that capital costs are correctly annuitised.""" + batteries_csv = """ + storage_name, lifetime, fom_$/kw/annum, connection_cost_$/mw, build_cost_$/mw, technology_specific_lcf_% + Battery_2hr_2024, 30, 15.0, 85000, 1950000, 100.0 + Battery_2hr_2026, 25, 20.0, 90000, 1400000, 95.0 + """ + batteries_df = csv_str_to_df(batteries_csv) + wacc = 0.05 # 5% weighted average cost of capital + + result = _calculate_annuitised_new_entrant_battery_capital_costs(batteries_df, wacc) + + expected_result_csv = """ + storage_name, lifetime, fom_$/kw/annum, connection_cost_$/mw, build_cost_$/mw, technology_specific_lcf_%, capital_cost + Battery_2hr_2024, 30, 15.0, 85000, 1950000, 100.0, 147379.67 + Battery_2hr_2026, 25, 20.0, 90000, 1400000, 95.0, 120752.49 + """ + expected_result = csv_str_to_df(expected_result_csv) + + pd.testing.assert_frame_equal( + result.reset_index(drop=True).sort_values("storage_name"), + expected_result.reset_index(drop=True).sort_values("storage_name"), + check_dtype=False, + check_exact=False, + atol=1e-2, + ) diff --git a/tests/test_translator_and_model/test_limit_on_transmission_expansion.py b/tests/test_translator_and_model/test_limit_on_transmission_expansion.py index de4a2e5a..553263c5 100644 --- a/tests/test_translator_and_model/test_limit_on_transmission_expansion.py +++ b/tests/test_translator_and_model/test_limit_on_transmission_expansion.py @@ -148,6 +148,8 @@ def test_flow_path_expansion_limit_respected(csv_str_to_df, tmp_path, monkeypatc "ecaa_generators": ecaa_generators, "new_entrant_generators": pd.DataFrame(), "renewable_energy_zones": pd.DataFrame(), + "ecaa_batteries": pd.DataFrame(), + "new_entrant_batteries": pd.DataFrame(), } # Create a ModelConfig instance diff --git a/tests/test_translator_and_model/test_time_varying_flow_path_costs.py b/tests/test_translator_and_model/test_time_varying_flow_path_costs.py index a486a25d..79923990 100644 --- a/tests/test_translator_and_model/test_time_varying_flow_path_costs.py +++ b/tests/test_translator_and_model/test_time_varying_flow_path_costs.py @@ -144,6 +144,8 @@ def test_link_expansion_economic_timing(csv_str_to_df, tmp_path, monkeypatch): "ecaa_generators": ecaa_generators, "new_entrant_generators": pd.DataFrame(), "renewable_energy_zones": pd.DataFrame(), + "ecaa_batteries": pd.DataFrame(), + "new_entrant_batteries": pd.DataFrame(), } # Create a ModelConfig instance diff --git a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py index 5628d0e4..3bc9b44c 100644 --- a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py +++ b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py @@ -184,6 +184,8 @@ def test_vre_build_limit_constraint(csv_str_to_df, tmp_path, monkeypatch): "renewable_energy_zones": renewable_energy_zones, "new_entrant_build_costs": new_entrant_build_costs, "new_entrant_wind_and_solar_connection_costs": new_entrant_wind_and_solar_connection_costs, + "ecaa_batteries": pd.DataFrame(), + "new_entrant_batteries": pd.DataFrame(), } # Create a ModelConfig instance diff --git a/tests/test_workbook_table_cache/battery_properties.csv b/tests/test_workbook_table_cache/battery_properties.csv new file mode 100644 index 00000000..0fe44610 --- /dev/null +++ b/tests/test_workbook_table_cache/battery_properties.csv @@ -0,0 +1,7 @@ +Property,Battery storage (1hr storage),Battery storage (2hrs storage),Battery storage (4hrs storage),Battery storage (8hrs storage),Units +Maximum power,1,1,1,1,MW +Energy capacity,1,2,4,8,MWh +Charge efficiency (utility),91.7,91.7,92.2,91.1,% +Discharge efficiency (utility),91.7,91.7,92.2,91.1,% +Round trip efficiency (utility),84,84,85,83,% +Annual degradation (utility),1.8,1.8,1.8,1.8,% From b9a48059ee830210ce6a351b9a13436bc7725b92 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 01:40:38 +1100 Subject: [PATCH 03/12] specify test checking for strings remaining after templating batteries --- tests/test_templater/test_storage.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_templater/test_storage.py b/tests/test_templater/test_storage.py index 99dc0417..3ea5ec33 100644 --- a/tests/test_templater/test_storage.py +++ b/tests/test_templater/test_storage.py @@ -28,6 +28,14 @@ def test_battery_templater(workbook_table_cache_test_path: Path): ecaa_batteries, new_entrant_batteries = _template_battery_properties(iasr_tables) for ecaa_property_col in _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP.keys(): + if ( + "new_col_name" + in _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP[ecaa_property_col].keys() + ): + ecaa_property_col = _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP[ + ecaa_property_col + ]["new_col_name"] + if ( ecaa_property_col in _MINIMUM_REQUIRED_BATTERY_COLUMNS and "date" not in ecaa_property_col @@ -41,6 +49,16 @@ def test_battery_templater(workbook_table_cache_test_path: Path): for ( new_entrant_property_col ) in _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP.keys(): + if ( + "new_col_name" + in _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP[ + new_entrant_property_col + ].keys() + ): + new_entrant_property_col = _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP[ + new_entrant_property_col + ]["new_col_name"] + if ( new_entrant_property_col in _MINIMUM_REQUIRED_BATTERY_COLUMNS and "date" not in new_entrant_property_col @@ -62,6 +80,26 @@ def test_battery_templater(workbook_table_cache_test_path: Path): assert column in _MINIMUM_REQUIRED_BATTERY_COLUMNS +def test_merge_and_set_battery_static_properties_string_handling( + workbook_table_cache_test_path: Path, +): + """Test that string values in numeric columns are properly handled when merging static properties.""" + iasr_tables = read_csvs(workbook_table_cache_test_path) + + # Get the original data + ecaa_batteries, new_entrant_batteries = _template_battery_properties(iasr_tables) + + # Check that string values were properly handled in static property columns + for df in [ecaa_batteries, new_entrant_batteries]: + numeric_cols = df.select_dtypes(include=["number"]).columns + for col in numeric_cols: + if col in df.columns and "date" not in col: + # Verify no string values remain in numeric columns + assert not df[col].apply(lambda x: isinstance(x, str)).any(), ( + f"Column {col} contains string values" + ) + + def test_add_closure_year_column(csv_str_to_df): """Test the _add_closure_year_column function with various scenarios.""" # Setup test data From 18874d635eab9d72f8d612688285f951e399c746 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 08:17:53 +1100 Subject: [PATCH 04/12] add test for battery filtering function --- tests/test_templater/test_filter_template.py | 45 ++++++++++++++++++++ tests/test_templater/test_storage.py | 9 ++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tests/test_templater/test_filter_template.py b/tests/test_templater/test_filter_template.py index bc7fa2a8..9b20fc78 100644 --- a/tests/test_templater/test_filter_template.py +++ b/tests/test_templater/test_filter_template.py @@ -5,6 +5,7 @@ from ispypsa.templater.filter_template import ( _determine_selected_regions, + _filter_batteries, _filter_custom_constraints, _filter_expansion_costs, _filter_generator_dependent_tables, @@ -247,6 +248,50 @@ def test_filter_generator_dependent_tables(csv_str_to_df): ) +def test_filter_batteries(csv_str_to_df): + # Input data + template = { + "ecaa_batteries": csv_str_to_df(""" + storage_name, sub_region_id, region_id, capacity_mw + Limondale__BESS, SNSW, NSW, 50 + Chinchilla__BESS, SQ, QLD, 100 + Dalrymple__BESS, CSA, SA, 30 + """), + "new_entrant_batteries": csv_str_to_df(""" + storage_name, sub_region_id, region_id, technology_type + Battery_1h_CNSW, CNSW, NSW, Battery__Storage__1h + Battery_2h_VIC, VIC, VIC, Battery__Storage__2h + """), + } + + # Filter to NSW sub-regions only + filtered, selected_batteries = _filter_batteries(template, ["CNSW", "SNSW"]) + + # Expected results + expected_ecaa = csv_str_to_df(""" + storage_name, sub_region_id, region_id, capacity_mw + Limondale__BESS, SNSW, NSW, 50 + """) + + expected_new_entrant = csv_str_to_df(""" + storage_name, sub_region_id, region_id, technology_type + Battery_1h_CNSW, CNSW, NSW, Battery__Storage__1h + """) + + # Compare results + pd.testing.assert_frame_equal( + filtered["ecaa_batteries"].sort_values("storage_name").reset_index(drop=True), + expected_ecaa.sort_values("storage_name").reset_index(drop=True), + ) + pd.testing.assert_frame_equal( + filtered["new_entrant_batteries"], expected_new_entrant + ) + assert selected_batteries == { + "Limondale BESS", + "Battery_1h_CNSW", + } + + def test_get_selected_rezs_and_flow_paths(csv_str_to_df): """Test extraction of REZ IDs and flow path names from filtered tables.""" # Input data diff --git a/tests/test_templater/test_storage.py b/tests/test_templater/test_storage.py index 3ea5ec33..838bfb77 100644 --- a/tests/test_templater/test_storage.py +++ b/tests/test_templater/test_storage.py @@ -526,9 +526,12 @@ def test_calculate_and_merge_tech_specific_lcfs( Battery__Storage__(2hrs__storage), Battery__Storage__(2hrs__storage), """ expected_missing = csv_str_to_df(expected_missing_csv) - # match nan types: - expected_missing = expected_missing.fillna(pd.NA) - result_missing = result_missing.fillna(pd.NA) + # try to match nan types: + expected_missing = expected_missing.fillna("replace_with_nan") + expected_missing = expected_missing.replace("replace_with_nan", pd.NA) + + result_missing = result_missing.fillna("replace_with_nan") + result_missing = result_missing.replace("replace_with_nan", pd.NA) pd.testing.assert_frame_equal( result_missing.sort_values("storage_name").reset_index(drop=True), From 2c51586cf3adcd1db210d902857c81ca1175e0d1 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 08:36:41 +1100 Subject: [PATCH 05/12] add tests to explicitly check carriers added to model correctly --- src/ispypsa/pypsa_build/carriers.py | 12 ++-- tests/test_model/test_add_carriers.py | 98 +++++++++++++++++++++++++++ tests/test_model/test_add_storage.py | 14 ---- 3 files changed, 105 insertions(+), 19 deletions(-) create mode 100644 tests/test_model/test_add_carriers.py diff --git a/src/ispypsa/pypsa_build/carriers.py b/src/ispypsa/pypsa_build/carriers.py index 268662a9..0bf0e82a 100644 --- a/src/ispypsa/pypsa_build/carriers.py +++ b/src/ispypsa/pypsa_build/carriers.py @@ -1,19 +1,21 @@ -from pathlib import Path - import pandas as pd import pypsa def _add_carriers_to_network( - network: pypsa.Network, generators: pd.DataFrame, storage: pd.DataFrame + network: pypsa.Network, + generators: pd.DataFrame | None, + storage: pd.DataFrame | None, ) -> None: """Adds the Carriers in the generators table, and the AC and DC Carriers to the `pypsa.Network`. Args: network: The `pypsa.Network` object - generators: `pd.DataFrame` with `PyPSA` style `Generator` attributes. - storage: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes. At the moment this comprises batteries only. + generators: `pd.DataFrame` with `PyPSA` style `Generator` attributes, or None if no + such table exists. + storage: `pd.DataFrame` with `PyPSA` style `StorageUnit` attributes, or None if no such + table exists. At the moment this comprises batteries only. Returns: None """ diff --git a/tests/test_model/test_add_carriers.py b/tests/test_model/test_add_carriers.py new file mode 100644 index 00000000..b9860541 --- /dev/null +++ b/tests/test_model/test_add_carriers.py @@ -0,0 +1,98 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.model.carriers import _add_carriers_to_network + + +@pytest.fixture +def mock_network(): + """Create a minimal PyPSA network for testing.""" + return pypsa.Network() + + +def test_add_carriers_to_network_with_data(mock_network, csv_str_to_df): + """Test adding carriers to network with generator and storage data.""" + # Create test generators DataFrame + generators_csv = """ + name, bus, carrier, p_nom + gen1, bus1, Wind, 100 + gen2, bus2, Solar, 200 + gen3, bus3, Gas, 300 + """ + generators = csv_str_to_df(generators_csv) + + # Create test storage DataFrame + storage_csv = """ + name, bus, carrier, p_nom, max_hours + bat1, bus1, Battery, 50, 4 + bat2, bus2, Battery, 75, 2 + """ + storage = csv_str_to_df(storage_csv) + + # Call function + _add_carriers_to_network(mock_network, generators, storage) + + # Check all expected carriers are added + expected_carriers = ["Wind", "Solar", "Gas", "Battery", "AC", "DC"] + assert set(mock_network.carriers.index) == set(expected_carriers) + + +def test_add_carriers_to_network_one_input_source_only(mock_network, csv_str_to_df): + """Test adding carriers to network with only generator data.""" + # Create test generators DataFrame + generators_csv = """ + name, bus, carrier, p_nom + gen1, bus1, Wind, 100 + gen2, bus2, Coal, 200 + """ + generators = csv_str_to_df(generators_csv) + + # Call function with no storage + _add_carriers_to_network(mock_network, generators, None) + + # Check expected carriers are added + expected_carriers = ["Wind", "Coal", "AC", "DC"] + assert set(mock_network.carriers.index) == set(expected_carriers) + + +def test_add_carriers_to_network_storage_only(mock_network, csv_str_to_df): + """Test adding carriers to network with only storage data.""" + # Create test storage DataFrame + storage_csv = """ + name, bus, carrier, p_nom, max_hours + bat1, bus1, Battery, 50, 4 + bat2, bus2, Flow, 75, 6 + """ + storage = csv_str_to_df(storage_csv) + + # Call function with no generators + _add_carriers_to_network(mock_network, None, storage) + + # Check expected carriers are added + expected_carriers = ["Battery", "Flow", "AC", "DC"] + assert set(mock_network.carriers.index) == set(expected_carriers) + + +def test_add_carriers_to_network_empty_dataframes(mock_network): + """Test adding carriers to network with empty DataFrames.""" + # Create empty DataFrames + generators = pd.DataFrame(columns=["carrier"]) + storage = pd.DataFrame(columns=["carrier"]) + + # Call function + _add_carriers_to_network(mock_network, generators, storage) + + # Check only standard carriers are added + expected_carriers = ["AC", "DC"] + assert set(mock_network.carriers.index) == set(expected_carriers) + + +def test_add_carriers_to_network_no_data(mock_network): + """Test adding carriers to network with no data.""" + # Call function with None for both parameters + _add_carriers_to_network(mock_network, None, None) + + # Check only standard carriers are added + expected_carriers = ["AC", "DC"] + assert set(mock_network.carriers.index) == set(expected_carriers) diff --git a/tests/test_model/test_add_storage.py b/tests/test_model/test_add_storage.py index 4e9d5f0a..92c448dd 100644 --- a/tests/test_model/test_add_storage.py +++ b/tests/test_model/test_add_storage.py @@ -14,20 +14,6 @@ def mock_network(csv_str_to_df): network = pypsa.Network() network.add("Bus", "test_bus") - # Create sample trace data - snapshots_csv = """ - investment_periods, snapshots - 2024, 2023-07-01__12:00:00 - 2024, 2024-04-01__12:00:00 - 2025, 2024-07-01__12:00:00 - 2025, 2025-04-01__12:00:00 - """ - snapshots = csv_str_to_df(snapshots_csv) - snapshots["snapshots"] = pd.to_datetime(snapshots["snapshots"]) - - # set network snapshots: - network.snapshots = snapshots - return network From 39f525e717f68866de924763c97838c1308596ed Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 09:18:07 +1100 Subject: [PATCH 06/12] remove redundant storage duration function from generator templater --- .../static_new_generator_properties.py | 25 +------------------ .../test_generator_static_properties.py | 18 +++++++------ 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/src/ispypsa/templater/static_new_generator_properties.py b/src/ispypsa/templater/static_new_generator_properties.py index 9a28a19b..06e02925 100644 --- a/src/ispypsa/templater/static_new_generator_properties.py +++ b/src/ispypsa/templater/static_new_generator_properties.py @@ -473,26 +473,6 @@ def _add_isp_resource_type_column( return df_with_isp_resource_type -def _add_storage_duration_column( - df: pd.DataFrame, storage_duration_col_name: str -) -> pd.DataFrame: - """Adds a new column to the new entrant generator table to hold storage duration in hours.""" - - # if 'storage' is present in the name -> grab the hours from the name string: - def _get_storage_duration(name: str) -> str | None: - duration_pattern = r"(?P\d+h)rs* storage" - duration_string = re.search(duration_pattern, name, re.IGNORECASE) - - if duration_string: - return duration_string.group("duration") - else: - return None - - df[storage_duration_col_name] = df["generator_name"].map(_get_storage_duration) - - return df - - def _add_unique_generator_string_column( df: pd.DataFrame, generator_string_col_name: str = "generator" ) -> pd.DataFrame: @@ -563,11 +543,8 @@ def _add_identifier_columns(df: pd.DataFrame, iasr_tables: dict[str, pd.DataFram df_with_rez_ids = _add_and_clean_rez_ids( df, "rez_id", iasr_tables["renewable_energy_zones"] ) - df_with_storage_duration = _add_storage_duration_column( - df_with_rez_ids, "storage_duration" - ) df_with_isp_resource_type = _add_isp_resource_type_column( - df_with_storage_duration, "isp_resource_type" + df_with_rez_ids, "isp_resource_type" ) df_with_unique_generator_str = _add_unique_generator_string_column( diff --git a/tests/test_templater/test_generator_static_properties.py b/tests/test_templater/test_generator_static_properties.py index 2d1ca1fa..d6676217 100644 --- a/tests/test_templater/test_generator_static_properties.py +++ b/tests/test_templater/test_generator_static_properties.py @@ -43,9 +43,15 @@ def test_static_ecaa_generator_templater(workbook_table_cache_test_path: Path): ("Existing", "Committed", "Anticipated", "Additional projects") ) + # check that columns present are all required columns: for column in df.columns: assert column in _MINIMUM_REQUIRED_GENERATOR_COLUMNS + # checks that all entries in "generator" col are strings + assert all(df.generator.apply(lambda x: True if isinstance(x, str) else False)) + # checks that all entries in "generator" col are unique + assert len(df.generator.unique()) == len(df.generator) + where_solar, where_wind = ( df["technology_type"].str.contains("solar", case=False), df["technology_type"].str.contains("wind", case=False), @@ -84,13 +90,13 @@ def test_static_new_generator_templater(workbook_table_cache_test_path: Path): # checks that all entries in "generator" col are strings assert all(df.generator.apply(lambda x: True if isinstance(x, str) else False)) + # check that all entries in "generator" col are unique + assert len(df.generator.unique()) == len(df.generator) # checks that values that should be always set to zero are zero: - where_solar, where_wind, where_hydro, where_battery, where_ocgt, where_h2 = ( + where_solar, where_wind, where_ocgt, where_h2 = ( df["generator"].str.contains("solar", case=False), df["generator"].str.contains("wind", case=False), - df["generator"].str.contains("pumped hydro", case=False), - df["generator"].str.contains("battery", case=False), df["generator"].str.contains("ocgt", case=False), df["generator"].str.contains("hydrogen", case=False), ) @@ -98,19 +104,15 @@ def test_static_new_generator_templater(workbook_table_cache_test_path: Path): "minimum_stable_level_%": ( where_solar, where_wind, - where_hydro, - where_battery, where_ocgt, where_h2, ), "vom_$/mwh_sent_out": ( where_solar, where_wind, - where_hydro, - where_battery, where_h2, ), - "heat_rate_gj/mwh": (where_solar, where_wind, where_hydro, where_battery), + "heat_rate_gj/mwh": (where_solar, where_wind), } for zero_col_name, technology_dfs in zero_tests.items(): for where_tech in technology_dfs: From d4dba27e22224f41d3b4d924f93d860198872198 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:12:26 +1100 Subject: [PATCH 07/12] add beginnings of generation and (battery) storage method --- docs/method.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/method.md b/docs/method.md index a97aaf88..6893ba5d 100644 --- a/docs/method.md +++ b/docs/method.md @@ -95,8 +95,36 @@ further information on custom constraint implementation. ## Generation +Generation is represented as a time-varying quantity for each generator: + +- Variable renewable energy (VRE) generation data traces are the generation data published by AEMO +for each project or REZ and resource type. These traces set the upper limit on VRE generator output in +each modelled snapshot. +- The historical weather years, or reference years, used as a basis for deriving the time varying +generation data are defined using the `reference_year_cycle` options in the config. [More detail on +reference years](#reference-years). +- The time varying quantity of generation at each node is also dependent on the model year. +AEMO publishes generation data for every year in modelling horizon for each reference year. +- Other non-VRE generation is currently modelled under static output limits set by each generator's +maximum capacity `maximum_capacity_mw` and minimum stable generation `minimum_load_mw` or `minimum_stable_level_%` (where defined). +- Generator dispatch is optimised at each snapshot to meet demand at the lowest cost while meeting +the output constraints described above. + ## Storage +Storage charging and discharging behaviour is also represented as a time-varying quantity for +each storage unit in the model: + +- Charging and discharging efficiencies are defined for each battery and applied to charge/discharge +in each snapshot accordingly. +- The state of charge of each battery in each snapshot is determined by the previous state of charge +plus energy charged minus energy discharged (with efficiencies applied). +- Charge and discharge power and energy in each snapshot are limited by the `maximum_capacity_mw` +and `maximum_capacity_mw` $\times$ `storage_duration_hours` properties of each battery, as well +as the available state of charge. +- Battery charging and discharging behaviour is optimised for each snapshot to meet demand at lowest cost, +while subject to energy balance constraints. + ## Reference years Weather reference years are used ensure weather correlations are consistent between demand @@ -237,6 +265,27 @@ modelled as relaxing the custom constraint limit. ### Generation +Generator capacities are decided by the model at the start of each [investment period](#investment-periodisation-and-discounting) +and for each [node](#nodal-representation): + +- The model currently considers all ECAA generators that are active (not retired) during the +model investment periods. These projects have set capacities and are not extendable during the capacity expansion modelling, +and have fixed retirement dates. +- New entrant generator are extendable in the model, and the optimisation determines the capacity of new generation to be built at each node in each investment period. +- Capital costs in $/MW for new entrant generators are annuitised according to the following formula: + + $$ + c_{a} =\frac{c_{o} \times r }{1 - (1 + r)^{-t}} + $$ + + Where $c_{a}$ is the annuitised cost and, $r$ is the [WACC](config.md#wacc), and $t$ is the generator lifetime. + $c_{o}$ includes the overnight build cost (adjusted by locational cost factors), any applicable connection costs + and/or additional system strength connection costs, and fixed operational costs. +- Marginal costs in $/MWh are calculated for all generators for each model snapshot based on +dynamic fuel prices, generator heat rates and variable operational costs defined in the input tables. Alternatively +a static value can be set for a subset or all generators to simplify the model. +- Where build or resource limit constraints are defined for VRE generation in specific REZs, these are set by custom constraints. Some resource limits can be relaxed up to the corresponding build limit for the specified resource type and REZ. + ## Operational Operational is the second modelling phase. In this modelling phase capacity expansion decisions are taken as fixed, From f2a12e6782b799380fc44f130a36beb5929f7865 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:18:22 +1100 Subject: [PATCH 08/12] fix up model to pypsa_build update in new storage locations --- src/ispypsa/pypsa_build/save.py | 2 +- src/ispypsa/pypsa_build/update.py | 2 +- tests/test_model/test_add_carriers.py | 2 +- tests/test_model/test_add_storage.py | 2 +- tests/test_translator/test_translate_storage.py | 6 ++++-- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ispypsa/pypsa_build/save.py b/src/ispypsa/pypsa_build/save.py index cfef389f..fd77ff53 100644 --- a/src/ispypsa/pypsa_build/save.py +++ b/src/ispypsa/pypsa_build/save.py @@ -11,7 +11,7 @@ def save_pypsa_network( Examples: Perform required imports. >>> from pathlib import Path - >>> from ispypsa.model import save_pypsa_network + >>> from ispypsa.pypsa_build import save_pypsa_network After running the model optimisation, save the network. >>> network.optimize.solve_model(solver_name="highs") diff --git a/src/ispypsa/pypsa_build/update.py b/src/ispypsa/pypsa_build/update.py index 891dd06b..72409fe7 100644 --- a/src/ispypsa/pypsa_build/update.py +++ b/src/ispypsa/pypsa_build/update.py @@ -25,7 +25,7 @@ def update_network_timeseries( >>> import pandas as pd >>> from pathlib import Path >>> from ispypsa.data_fetch import read_csvs - >>> from ispypsa.model import update_network_timeseries + >>> from ispypsa.pypsa_build import update_network_timeseries Get PyPSA friendly inputs (inparticular these need to contain the generators and buses tables). diff --git a/tests/test_model/test_add_carriers.py b/tests/test_model/test_add_carriers.py index b9860541..776f704c 100644 --- a/tests/test_model/test_add_carriers.py +++ b/tests/test_model/test_add_carriers.py @@ -2,7 +2,7 @@ import pypsa import pytest -from ispypsa.model.carriers import _add_carriers_to_network +from ispypsa.pypsa_build.carriers import _add_carriers_to_network @pytest.fixture diff --git a/tests/test_model/test_add_storage.py b/tests/test_model/test_add_storage.py index 92c448dd..a877af7b 100644 --- a/tests/test_model/test_add_storage.py +++ b/tests/test_model/test_add_storage.py @@ -2,7 +2,7 @@ import pypsa import pytest -from ispypsa.model.storage import ( +from ispypsa.pypsa_build.storage import ( _add_batteries_to_network, _add_battery_to_network, ) diff --git a/tests/test_translator/test_translate_storage.py b/tests/test_translator/test_translate_storage.py index ee1aced7..23d9cc72 100644 --- a/tests/test_translator/test_translate_storage.py +++ b/tests/test_translator/test_translate_storage.py @@ -128,12 +128,14 @@ def test_translate_ecaa_batteries_lifetime_calculation(csv_str_to_df): investment_periods = [2025] result = _translate_ecaa_batteries(ispypsa_tables, investment_periods) + print(result) # Battery3 should be filtered out (closure_year < investment_period) + # build_year now clipped so earliest possible build year is investment_period[0], i.e. 2025 here expected_result_csv = """ name, bus, p_nom, p_nom_extendable, carrier, max_hours, capital_cost, build_year, lifetime, efficiency_store, efficiency_dispatch, isp_resource_type, isp_rez_id - Battery1, CNSW, 100.0, False, Battery, 4, 0.0, 2020, 15, 0.95, 0.95, Battery__Storage__4h, - Battery2, CNSW, 200.0, False, Battery, 2, 0.0, 2023, 20, 0.90, 0.90, Battery__Storage__2h, + Battery1, CNSW, 100.0, False, Battery, 4, 0.0, 2025, 15, 0.95, 0.95, Battery__Storage__4h, + Battery2, CNSW, 200.0, False, Battery, 2, 0.0, 2025, 20, 0.90, 0.90, Battery__Storage__2h, """ expected_result = csv_str_to_df(expected_result_csv) From 77d3741bcc9951b728717581337420e6c8fffb83 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:21:26 +1100 Subject: [PATCH 09/12] apply review suggestions and fix efficiency values --- src/ispypsa/templater/mappings.py | 5 --- src/ispypsa/templater/storage.py | 2 - src/ispypsa/translator/mappings.py | 3 +- src/ispypsa/translator/storage.py | 21 +++++++++ tests/conftest.py | 8 ++-- .../test_translator/test_translate_storage.py | 45 ++++++++++++------- ...test_vre_build_limit_custom_constraints.py | 2 - 7 files changed, 54 insertions(+), 32 deletions(-) diff --git a/src/ispypsa/templater/mappings.py b/src/ispypsa/templater/mappings.py index a1d8cb9a..831093c6 100644 --- a/src/ispypsa/templater/mappings.py +++ b/src/ispypsa/templater/mappings.py @@ -348,11 +348,6 @@ } _NEW_ENTRANT_STORAGE_STATIC_PROPERTY_TABLE_MAP = { - "maximum_capacity_mw": dict( - table="maximum_capacity_new_entrants", - table_lookup="Generator type", - table_value="Total plant size (MW)", - ), "fom_$/kw/annum": dict( table="fixed_opex_new_entrants", table_lookup="Generator", diff --git a/src/ispypsa/templater/storage.py b/src/ispypsa/templater/storage.py index f113bcdf..0a5d3f16 100644 --- a/src/ispypsa/templater/storage.py +++ b/src/ispypsa/templater/storage.py @@ -19,8 +19,6 @@ _NEW_STORAGE_NEW_COLUMN_MAPPING, ) -pd.set_option("display.max_columns", None) - def _template_battery_properties( iasr_tables: dict[str, pd.DataFrame], diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index 8b27cda1..ed6e25b2 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -21,9 +21,7 @@ # attributes used by the PyPSA network model: "generator": "name", "p_nom": "p_nom", - # "unit_capacity_mw": "p_nom_mod", "p_nom_extendable": "p_nom_extendable", - # "maximum_capacity_mw": "p_nom_max", "minimum_stable_level_%": "p_min_pu", "fuel_type": "carrier", "marginal_cost": "marginal_cost", @@ -109,6 +107,7 @@ "capital_cost", "build_year", "lifetime", + "cyclic_state_of_charge", "efficiency_store", "efficiency_dispatch", "isp_resource_type", diff --git a/src/ispypsa/translator/storage.py b/src/ispypsa/translator/storage.py index 4d2399b4..0fa9e2f7 100644 --- a/src/ispypsa/translator/storage.py +++ b/src/ispypsa/translator/storage.py @@ -89,6 +89,17 @@ def _translate_ecaa_batteries( :, battery_attributes.keys() ].rename(columns=battery_attributes) + # Now make sure that if charge/discharge efficiency are present (they should be) + # that they're floats between 0-1: + efficiency_cols = [ + col for col in ecaa_batteries_pypsa_format.columns if "efficiency" in col + ] + for col in efficiency_cols: + ecaa_batteries_pypsa_format[col] /= 100 + + # For all batteries set cyclic_state_of_charge to True + ecaa_batteries_pypsa_format["cyclic_state_of_charge"] = True + columns_in_order = [ col for col in _BATTERY_ATTRIBUTE_ORDER @@ -186,6 +197,16 @@ def _translate_new_entrant_batteries( :, battery_attributes.keys() ].rename(columns=battery_attributes) + # enforce efficiency values are floats between 0-1 + efficiency_cols = [ + col for col in new_entrant_batteries_pypsa_format.columns if "efficiency" in col + ] + for col in efficiency_cols: + new_entrant_batteries_pypsa_format[col] /= 100 + + # For all batteries set cyclic_state_of_charge to True + new_entrant_batteries_pypsa_format["cyclic_state_of_charge"] = True + columns_in_order = [ col for col in _BATTERY_ATTRIBUTE_ORDER diff --git a/tests/conftest.py b/tests/conftest.py index 48510fe5..5812d28e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -255,16 +255,16 @@ def sample_ispypsa_tables(csv_str_to_df): # ECAA battery table: ecaa_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type - Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h - Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 95.0, 95.0, 4, Battery, Battery__Storage__4h + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 90.0, 90.0, 2, Battery, Battery__Storage__2h """ tables["ecaa_batteries"] = csv_str_to_df(ecaa_batteries_csv) # New entrant battery table: new_entrant_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type - NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h - NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 90.0, 90.0, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 95.0, 95.0, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h """ tables["new_entrant_batteries"] = csv_str_to_df(new_entrant_batteries_csv) diff --git a/tests/test_translator/test_translate_storage.py b/tests/test_translator/test_translate_storage.py index 23d9cc72..7ceeefb4 100644 --- a/tests/test_translator/test_translate_storage.py +++ b/tests/test_translator/test_translate_storage.py @@ -36,8 +36,8 @@ def test_translate_ecaa_batteries_basic(csv_str_to_df): # Create test input data ecaa_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type - Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h - Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 95.0, 95.0, 4, Battery, Battery__Storage__4h + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 90.0, 90.0, 2, Battery, Battery__Storage__2h """ ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} @@ -49,9 +49,15 @@ def test_translate_ecaa_batteries_basic(csv_str_to_df): for templater_name, translated_name in _ECAA_BATTERY_ATTRIBUTES.items(): assert translated_name in result.columns + # check that efficiency values are all between 0-1: + efficiency_cols = [col for col in result.columns if "efficiency" in col] + for col in efficiency_cols: + assert result[col].between(0, 1).all() + assert "bus" in result.columns # check this separately (not in mapping) assert not result["p_nom_extendable"].any() # All should be False assert (result["capital_cost"] == 0.0).all() # All should be 0.0 + assert all(result["cyclic_state_of_charge"]) # All should be True # Check that no extra columns were added: extra_cols = set(result.columns) - set(_BATTERY_ATTRIBUTE_ORDER) @@ -63,7 +69,7 @@ def test_translate_ecaa_batteries_regional_granularity(csv_str_to_df): # Create test input data ecaa_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type - Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + Battery2, NNSW, NSW, , 2022-07-1, 2042, 200, 90.0, 90.0, 2, Battery, Battery__Storage__2h """ ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} @@ -93,8 +99,8 @@ def test_translate_ecaa_batteries_rez_handling(csv_str_to_df): # Create test input with REZ ecaa_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type - Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h - Battery2, CNSW, NSW, N3, 2022-07-01, 2042, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 95.0, 95.0, 4, Battery, Battery__Storage__4h + Battery2, CNSW, NSW, N3, 2022-07-01, 2042, 200, 90.0, 90.0, 2, Battery, Battery__Storage__2h """ ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} @@ -119,23 +125,22 @@ def test_translate_ecaa_batteries_lifetime_calculation(csv_str_to_df): # Create test input ecaa_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, commissioning_date, closure_year, maximum_capacity_mw, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, isp_resource_type - Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 0.95, 0.95, 4, Battery, Battery__Storage__4h - Battery2, CNSW, NSW, , 2022-07-01, 2045, 200, 0.90, 0.90, 2, Battery, Battery__Storage__2h - Battery3, CNSW, NSW, , 2010-04-01, 2020, 300, 0.92, 0.92, 8, Battery, Battery__Storage__8h + Battery1, CNSW, NSW, , 2020-01-01, 2040, 100, 95.0, 95.0, 4, Battery, Battery__Storage__4h + Battery2, CNSW, NSW, , 2022-07-01, 2045, 200, 90.0, 90.0, 2, Battery, Battery__Storage__2h + Battery3, CNSW, NSW, , 2010-04-01, 2020, 300, 92.0, 92.0, 8, Battery, Battery__Storage__8h """ ispypsa_tables = {"ecaa_batteries": csv_str_to_df(ecaa_batteries_csv)} investment_periods = [2025] result = _translate_ecaa_batteries(ispypsa_tables, investment_periods) - print(result) # Battery3 should be filtered out (closure_year < investment_period) # build_year now clipped so earliest possible build year is investment_period[0], i.e. 2025 here expected_result_csv = """ - name, bus, p_nom, p_nom_extendable, carrier, max_hours, capital_cost, build_year, lifetime, efficiency_store, efficiency_dispatch, isp_resource_type, isp_rez_id - Battery1, CNSW, 100.0, False, Battery, 4, 0.0, 2025, 15, 0.95, 0.95, Battery__Storage__4h, - Battery2, CNSW, 200.0, False, Battery, 2, 0.0, 2025, 20, 0.90, 0.90, Battery__Storage__2h, + name, bus, p_nom, p_nom_extendable, carrier, max_hours, capital_cost, build_year, lifetime, cyclic_state_of_charge, efficiency_store, efficiency_dispatch, isp_resource_type, isp_rez_id + Battery1, CNSW, 100.0, False, Battery, 4, 0.0, 2025, 15, True, 0.95, 0.95, Battery__Storage__4h, + Battery2, CNSW, 200.0, False, Battery, 2, 0.0, 2025, 20, True, 0.90, 0.90, Battery__Storage__2h, """ expected_result = csv_str_to_df(expected_result_csv) @@ -169,7 +174,7 @@ def test_translate_new_entrant_batteries_basic(csv_str_to_df, sample_ispypsa_tab # Create test input data new_entrant_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type - NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 90.0, 90.0, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h """ ispypsa_tables = { @@ -185,9 +190,15 @@ def test_translate_new_entrant_batteries_basic(csv_str_to_df, sample_ispypsa_tab for templater_name, translated_name in _NEW_ENTRANT_BATTERY_ATTRIBUTES.items(): assert translated_name in result.columns + # check that efficiency values are all between 0-1: + efficiency_cols = [col for col in result.columns if "efficiency" in col] + for col in efficiency_cols: + assert result[col].between(0, 1).all() + assert "bus" in result.columns # check this separately (it's not in mapping) assert result["p_nom_extendable"].all() # All should be True assert len(result) == 2 + assert all(result["cyclic_state_of_charge"]) # All should be True def test_translate_new_entrant_batteries_regional_granularity( @@ -196,8 +207,8 @@ def test_translate_new_entrant_batteries_regional_granularity( """Test different regional granularity settings for new entrant batteries.""" new_entrant_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type - NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h - NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 90.0, 90.0, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 95.0, 95.0, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h """ ispypsa_tables = { @@ -232,8 +243,8 @@ def test_translate_new_entrant_batteries_rez_handling( """Test REZ handling options for new entrant batteries.""" new_entrant_batteries_csv = """ storage_name, sub_region_id, region_id, rez_id, technology_type, lifetime, charging_efficiency_%, discharging_efficiency_%, storage_duration_hours, fuel_type, technology_specific_lcf_%, connection_cost_$/mw, fom_$/kw/annum, isp_resource_type - NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 0.90, 0.90, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h - NewBattery2, CNSW, NSW, N3, Battery__Storage__(2hrs__storage), 20, 0.95, 0.95, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h + NewBattery1, CNSW, NSW, , Battery__Storage__(2hrs__storage), 20, 90.0, 90.0, 2, Battery, 100.0, 55000.0, 10.0, Battery__Storage__2h + NewBattery2, CNSW, NSW, N3, Battery__Storage__(2hrs__storage), 20, 95.0, 95.0, 2, Battery, 100.0, 55000.0, 7.0, Battery__Storage__2h """ ispypsa_tables = { diff --git a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py index 3bc9b44c..ba7f9d19 100644 --- a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py +++ b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py @@ -9,8 +9,6 @@ create_pypsa_friendly_timeseries_inputs, ) -pd.set_option("display.max_columns", None) - def test_vre_build_limit_constraint(csv_str_to_df, tmp_path, monkeypatch): """Test that capacity expansion of VRE is limited by custom constraints in a REZ. From b36a6cc71d2e288bd1895c7fb2ff5ce67046ef18 Mon Sep 17 00:00:00 2001 From: nick-gorman <40549624+nick-gorman@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:38:07 +1100 Subject: [PATCH 10/12] battery plotting --- src/ispypsa/plotting/generation.py | 80 +++++++++-- src/ispypsa/plotting/plot.py | 9 +- src/ispypsa/plotting/style.py | 42 +++--- src/ispypsa/plotting/transmission.py | 5 +- src/ispypsa/plotting/website.py | 7 +- src/ispypsa/results/generation.py | 187 ++++++++++++++++++++++---- tests/test_results/test_generation.py | 113 +++++++++++++++- 7 files changed, 383 insertions(+), 60 deletions(-) diff --git a/src/ispypsa/plotting/generation.py b/src/ispypsa/plotting/generation.py index 0022ea2a..41b07291 100644 --- a/src/ispypsa/plotting/generation.py +++ b/src/ispypsa/plotting/generation.py @@ -349,16 +349,59 @@ def _create_export_trace(timesteps: pd.DatetimeIndex, values: list) -> go.Scatte ) +def _create_battery_charging_trace( + timesteps: pd.DatetimeIndex, values: list +) -> go.Scatter: + """Create a Plotly scatter trace for battery charging (shown as negative).""" + return go.Scatter( + x=timesteps, + y=values, # Negative to show charging consumes power + name="Battery Charging", + mode="lines", + stackgroup="two", + fillcolor=get_fuel_type_color("Battery Charging"), + line=dict(width=0), + legendgroup="Load", # Appears in Load legend group + legendgrouptitle_text="Load", + visible="legendonly", + hovertemplate="Battery Charging
%{y:.2f} MW", + ) + + def _create_plotly_figure( dispatch: pd.DataFrame, demand: pd.Series, title: str, transmission: pd.DataFrame | None = None, ) -> go.Figure: - """Create a Plotly figure with generation, demand, and optionally transmission.""" + """Create a Plotly figure with generation, demand, storage, and optionally transmission. + + Battery storage is split into discharging (positive, stacks with generation) + and charging (negative, stacks with load/exports). + """ fig = go.Figure() - # Add transmission traces if provided + # Separate battery dispatch from other generation + battery_dispatch = dispatch[dispatch["fuel_type"] == "Battery"].copy() + non_battery_dispatch = dispatch[dispatch["fuel_type"] != "Battery"].copy() + + # Prepare battery charging/discharging data + has_battery_data = not battery_dispatch.empty + if has_battery_data: + # Aggregate battery dispatch by timestep + battery_by_timestep = ( + battery_dispatch.groupby("timestep")["dispatch_mw"].sum().reset_index() + ) + # Discharging = positive values + battery_discharging = battery_by_timestep.copy() + battery_discharging["dispatch_mw"] = battery_discharging["dispatch_mw"].clip( + lower=0 + ) + # Charging = negative values (keep as negative for display) + battery_charging = battery_by_timestep.copy() + battery_charging["dispatch_mw"] = battery_charging["dispatch_mw"].clip(upper=0) + + # Add transmission traces if provided (hidden offset trace first) if transmission is not None and not transmission.empty: fig.add_trace( _create_generation_trace( @@ -375,14 +418,26 @@ def _create_plotly_figure( ) ) - # Add generation traces (sorted alphabetically) - fuel_types = sorted(dispatch["fuel_type"].unique()) + # Add battery discharging trace (stacks with generation) + if has_battery_data and battery_discharging["dispatch_mw"].sum() > 0: + fig.add_trace( + _create_generation_trace( + "Battery Discharging", + battery_discharging["timestep"], + battery_discharging["dispatch_mw"], + ) + ) + + # Add generation traces (sorted alphabetically, excluding Battery) + fuel_types = sorted(non_battery_dispatch["fuel_type"].unique()) for fuel_type in fuel_types: fig.add_trace( _create_generation_trace( fuel_type, - dispatch["timestep"], - dispatch[dispatch["fuel_type"] == fuel_type]["dispatch_mw"], + non_battery_dispatch["timestep"], + non_battery_dispatch[non_battery_dispatch["fuel_type"] == fuel_type][ + "dispatch_mw" + ], ) ) @@ -394,10 +449,19 @@ def _create_plotly_figure( ) ) + # Add battery charging trace (stacks with load/exports, shown as negative) + if has_battery_data and battery_charging["dispatch_mw"].sum() < 0: + fig.add_trace( + _create_battery_charging_trace( + battery_charging["timestep"], + battery_charging["dispatch_mw"], # Already negative + ) + ) + fig.add_trace(_create_demand_trace(demand["timestep"], demand["demand_mw"])) - # Apply professional styling - layout = create_plotly_professional_layout(title=title) + # Apply professional styling with timeseries formatting + layout = create_plotly_professional_layout(title=title, timeseries=True) fig.update_layout(**layout) return fig diff --git a/src/ispypsa/plotting/plot.py b/src/ispypsa/plotting/plot.py index e48b6c3a..3ed3a986 100644 --- a/src/ispypsa/plotting/plot.py +++ b/src/ispypsa/plotting/plot.py @@ -180,5 +180,10 @@ def save_plots(charts: dict[Path, dict], base_path: Path) -> None: csv_path = html_path.with_suffix(".csv") content["data"].to_csv(csv_path, index=False) - # Save the plot (HTML) - plot.write_html(html_path) + # Save the plot (HTML) with responsive sizing + plot.write_html( + html_path, + full_html=True, + include_plotlyjs=True, + config={"responsive": True}, + ) diff --git a/src/ispypsa/plotting/style.py b/src/ispypsa/plotting/style.py index 478a603b..ad067bec 100644 --- a/src/ispypsa/plotting/style.py +++ b/src/ispypsa/plotting/style.py @@ -21,6 +21,10 @@ # Hydrogen "Hydrogen": "#DDA0DD", "Hyblend": "#DDA0DD", + # Battery Storage + "Battery": "#3245c9", + "Battery Charging": "#577CFF", + "Battery Discharging": "#3245c9", # Transmission (for plotting) "Transmission Exports": "#927BAD", "Transmission Imports": "#521986", @@ -50,6 +54,7 @@ def create_plotly_professional_layout( title: str, height: int = 600, width: int = 1200, + timeseries: bool = False, ) -> dict: """Create professional/academic style layout for Plotly charts. @@ -57,12 +62,29 @@ def create_plotly_professional_layout( title: Chart title y_max: Maximum y-axis value y_min: Minimum y-axis value - height: Chart height in pixels - width: Chart width in pixels + height: Chart height in pixels (used as minimum height) + width: Chart width in pixels (ignored when autosize is True) + timeseries: If True, applies timeseries-specific formatting (rotated x-axis labels) Returns: Plotly layout dictionary """ + xaxis_config = { + "gridcolor": "#E0E0E0", + "gridwidth": 0.5, + "showgrid": True, + "showline": True, + "linewidth": 1, + "linecolor": "#CCCCCC", + "mirror": True, + "ticks": "outside", + "tickfont": {"size": 11}, + } + + if timeseries: + xaxis_config["tickformat"] = "%Y-%m-%d %H:%M" + xaxis_config["tickangle"] = 45 + return { "title": { "text": title, @@ -93,18 +115,7 @@ def create_plotly_professional_layout( "borderwidth": 1, "font": {"size": 11}, }, - "xaxis": { - "gridcolor": "#E0E0E0", - "gridwidth": 0.5, - "showgrid": True, - "showline": True, - "linewidth": 1, - "linecolor": "#CCCCCC", - "mirror": True, - "ticks": "outside", - "tickfont": {"size": 11}, - "tickformat": "%Y-%m-%d %H:%M", - }, + "xaxis": xaxis_config, "yaxis": { "gridcolor": "#E0E0E0", "gridwidth": 0.5, @@ -118,7 +129,6 @@ def create_plotly_professional_layout( "rangemode": "tozero", "tickformat": ",", # Comma separator }, - "height": height, - "width": width, + "autosize": True, "margin": {"l": 80, "r": 200, "t": 80, "b": 60}, } diff --git a/src/ispypsa/plotting/transmission.py b/src/ispypsa/plotting/transmission.py index 4c348fde..a1219d27 100644 --- a/src/ispypsa/plotting/transmission.py +++ b/src/ispypsa/plotting/transmission.py @@ -317,9 +317,10 @@ def plot_flows( ) ) - # Apply professional styling + # Apply professional styling with timeseries formatting layout = create_plotly_professional_layout( - title=f"{isp_name} - Week {week_starting} (Investment Period {investment_period})" + title=f"{isp_name} - Week {week_starting} (Investment Period {investment_period})", + timeseries=True, ) layout["yaxis_title"] = {"text": "Flow (MW)", "font": {"size": 14}} layout["xaxis_title"] = {"text": "Timestep", "font": {"size": 14}} diff --git a/src/ispypsa/plotting/website.py b/src/ispypsa/plotting/website.py index c340e2b0..a3e28b53 100644 --- a/src/ispypsa/plotting/website.py +++ b/src/ispypsa/plotting/website.py @@ -197,10 +197,11 @@ def _generate_html_template( .plot-container {{ flex: 1; display: flex; - align-items: center; + align-items: stretch; justify-content: center; - padding: 2rem; - overflow: auto; + padding: 1rem; + overflow: hidden; + min-height: 0; }} .plot-frame {{ diff --git a/src/ispypsa/results/generation.py b/src/ispypsa/results/generation.py index baf239d5..8013548e 100644 --- a/src/ispypsa/results/generation.py +++ b/src/ispypsa/results/generation.py @@ -5,20 +5,24 @@ def extract_generator_dispatch(network: pypsa.Network) -> pd.DataFrame: - """Extract generator dispatch data from PyPSA network. + """Extract generator and storage dispatch data from PyPSA network. + + Combines generator and storage unit dispatch into a single DataFrame. + For storage units, positive dispatch_mw means discharging (generating power), + negative means charging (consuming power). Args: network: PyPSA network with solved optimization results Returns: DataFrame with columns: - - generator: Name of the generator - - node: Bus/sub-region where generator is located - - fuel_type: Technology type (Solar, Wind, Gas, etc.) + - generator: Name of the generator or storage unit + - node: Bus/sub-region where generator/storage is located + - fuel_type: Technology type (Solar, Wind, Gas, Battery, etc.) - technology_type: ISP technology classification - investment_period: Investment period/year - timestep: Datetime of dispatch - - dispatch_mw: Power output in MW + - dispatch_mw: Power output in MW (negative for storage charging) """ # Get generator static data generators = network.generators[["bus", "carrier", "isp_technology_type"]].copy() @@ -46,48 +50,117 @@ def extract_generator_dispatch(network: pypsa.Network) -> pd.DataFrame: } ) + # Extract storage dispatch and combine + storage_dispatch = _extract_storage_dispatch(network) + # Reorder columns - dispatch_long = dispatch_long[ - [ - "generator", - "node", - "fuel_type", - "technology_type", - "investment_period", - "timestep", - "dispatch_mw", - ] + cols = [ + "generator", + "node", + "fuel_type", + "technology_type", + "investment_period", + "timestep", + "dispatch_mw", ] + dispatch_long = dispatch_long[cols] + + if not storage_dispatch.empty: + results = pd.concat([dispatch_long, storage_dispatch[cols]], ignore_index=True) + else: + results = dispatch_long - return dispatch_long + return results + + +def _extract_storage_dispatch(network: pypsa.Network) -> pd.DataFrame: + """Extract storage unit dispatch data from PyPSA network. + + Args: + network: PyPSA network with solved optimization results + + Returns: + DataFrame with columns matching generator dispatch: + - generator: Name of the storage unit + - node: Bus/sub-region where storage is located + - fuel_type: Carrier (typically "Battery") + - technology_type: Set to "Battery Storage" + - investment_period: Investment period/year + - timestep: Datetime of dispatch + - dispatch_mw: Power in MW (positive = discharging, negative = charging) + """ + if network.storage_units.empty or network.storage_units_t.p.empty: + return pd.DataFrame() + + # Get storage static data + storage_units = network.storage_units[["bus", "carrier"]].copy() + + # Get dispatch time series + storage_dispatch_t = network.storage_units_t.p.copy() + + # Reshape dispatch data from wide to long format + storage_dispatch_long = storage_dispatch_t.stack().reset_index() + storage_dispatch_long.columns = [ + "period", + "timestep", + "storage_unit_name", + "dispatch_mw", + ] + + # Merge with storage static data + storage_dispatch_long = storage_dispatch_long.merge( + storage_units, left_on="storage_unit_name", right_index=True, how="inner" + ) + + # Rename columns to match generator format + storage_dispatch_long = storage_dispatch_long.rename( + columns={ + "storage_unit_name": "generator", + "bus": "node", + "carrier": "fuel_type", + "period": "investment_period", + } + ) + + # Set technology_type to "Battery Storage" + storage_dispatch_long["technology_type"] = "Battery Storage" + + return storage_dispatch_long def extract_generation_expansion_results(network: pypsa.Network) -> pd.DataFrame: - """Extract generation expansion results from PyPSA network. + """Extract generation and storage expansion results from PyPSA network. + + Combines generator and storage unit capacity data into a single DataFrame. Args: network: PyPSA network with solved optimization results Returns: DataFrame with columns: - - generator: Name of the generator + - generator: Name of the generator or storage unit - fuel_type: Fuel type (carrier) - technology_type: ISP technology classification - - node: Bus/sub-region where generator is located + - node: Bus/sub-region where generator/storage is located - capacity_mw: Optimized capacity in MW - investment_period: Build year - - closure_year: Year when generator closes (build_year + lifetime) + - closure_year: Year when generator/storage closes (build_year + lifetime) """ - results = network.generators.copy() + # Extract generator results + generator_results = network.generators.copy() # Filter out constraint dummy generators - results = results[results["bus"] != "bus_for_custom_constraint_gens"] + generator_results = generator_results[ + generator_results["bus"] != "bus_for_custom_constraint_gens" + ] # Calculate closure_year - results["closure_year"] = results["build_year"] + results["lifetime"] + generator_results["closure_year"] = ( + generator_results["build_year"] + generator_results["lifetime"] + ) # Rename columns - results = results.rename( + generator_results = generator_results.rename( columns={ "carrier": "fuel_type", "isp_technology_type": "technology_type", @@ -98,9 +171,14 @@ def extract_generation_expansion_results(network: pypsa.Network) -> pd.DataFrame ) # Reset index to get generator name as column - results = results.reset_index().rename(columns={"Generator": "generator"}) + generator_results = generator_results.reset_index().rename( + columns={"Generator": "generator"} + ) + + # Extract storage unit results + storage_results = _extract_storage_expansion_results(network) - # Select and order columns + # Combine generator and storage results cols = [ "generator", "fuel_type", @@ -110,11 +188,66 @@ def extract_generation_expansion_results(network: pypsa.Network) -> pd.DataFrame "investment_period", "closure_year", ] - results = results[cols] + + generator_results = generator_results[cols] + + if not storage_results.empty: + results = pd.concat( + [generator_results, storage_results[cols]], ignore_index=True + ) + else: + results = generator_results return results +def _extract_storage_expansion_results(network: pypsa.Network) -> pd.DataFrame: + """Extract storage unit expansion results from PyPSA network. + + Args: + network: PyPSA network with solved optimization results + + Returns: + DataFrame with columns matching generator expansion results: + - generator: Name of the storage unit + - fuel_type: Fuel type (carrier - typically "Battery") + - technology_type: Set to "Battery Storage" + - node: Bus/sub-region where storage is located + - capacity_mw: Optimized capacity in MW (p_nom_opt) + - investment_period: Build year + - closure_year: Year when storage closes (build_year + lifetime) + """ + if network.storage_units.empty: + return pd.DataFrame() + + storage_results = network.storage_units.copy() + + # Calculate closure_year + storage_results["closure_year"] = ( + storage_results["build_year"] + storage_results["lifetime"] + ) + + # Rename columns to match generator format + storage_results = storage_results.rename( + columns={ + "carrier": "fuel_type", + "bus": "node", + "p_nom_opt": "capacity_mw", + "build_year": "investment_period", + } + ) + + # Set technology_type to "Battery Storage" since storage units don't have isp_technology_type + storage_results["technology_type"] = "Battery Storage" + + # Reset index to get storage unit name as column (named "generator" for consistency) + storage_results = storage_results.reset_index().rename( + columns={"StorageUnit": "generator"} + ) + + return storage_results + + def extract_demand(network: pypsa.Network) -> pd.DataFrame: """Extract demand/load data from PyPSA network. diff --git a/tests/test_results/test_generation.py b/tests/test_results/test_generation.py index 8950ccc7..6954dd7d 100644 --- a/tests/test_results/test_generation.py +++ b/tests/test_results/test_generation.py @@ -38,10 +38,12 @@ def test_extract_generator_dispatch(csv_str_to_df): # Set multi-index to match PyPSA structure dispatch_t = dispatch_df.set_index(["period", "timestep"]) - # 3. Mock Network + # 3. Mock Network with empty storage units network = Mock() network.generators = generators network.generators_t.p = dispatch_t + network.storage_units = pd.DataFrame() # Empty storage units + network.storage_units_t.p = pd.DataFrame() # Empty storage dispatch # 4. Expected Output # - custom_gen should be removed @@ -68,6 +70,69 @@ def test_extract_generator_dispatch(csv_str_to_df): pd.testing.assert_frame_equal(result, expected_df) +def test_extract_generator_dispatch_with_storage(csv_str_to_df): + """Test extraction of generator and storage dispatch results combined.""" + + # 1. Setup Generator Data + generators_csv = """ + name, bus, carrier, isp_technology_type + gen1, NSW1, Gas, Gas - CCGT + """ + generators = csv_str_to_df(generators_csv).set_index("name") + + # 2. Setup Generator Dispatch Data + gen_dispatch_csv = """ + period, timestep, gen1 + 2030, 2030-01-01 00:00:00, 100 + 2030, 2030-01-01 01:00:00, 110 + """ + gen_dispatch_t = csv_str_to_df(gen_dispatch_csv).set_index(["period", "timestep"]) + + # 3. Setup Storage Unit Data + storage_csv = """ + name, bus, carrier + battery1, NSW1, Battery + """ + storage_units = csv_str_to_df(storage_csv).set_index("name") + + # 4. Setup Storage Dispatch Data (positive = discharging, negative = charging) + storage_dispatch_csv = """ + period, timestep, battery1 + 2030, 2030-01-01 00:00:00, 50 + 2030, 2030-01-01 01:00:00, -30 + """ + storage_dispatch_t = csv_str_to_df(storage_dispatch_csv).set_index( + ["period", "timestep"] + ) + + # 5. Mock Network + network = Mock() + network.generators = generators + network.generators_t.p = gen_dispatch_t + network.storage_units = storage_units + network.storage_units_t.p = storage_dispatch_t + + # 6. Expected Output - generators and storage combined + expected_csv = """ + generator, node, fuel_type, technology_type, investment_period, timestep, dispatch_mw + gen1, NSW1, Gas, Gas - CCGT, 2030, 2030-01-01 00:00:00, 100 + gen1, NSW1, Gas, Gas - CCGT, 2030, 2030-01-01 01:00:00, 110 + battery1, NSW1, Battery, Battery Storage, 2030, 2030-01-01 00:00:00, 50 + battery1, NSW1, Battery, Battery Storage, 2030, 2030-01-01 01:00:00, -30 + """ + expected_df = csv_str_to_df(expected_csv) + + # 7. Run Function + result = extract_generator_dispatch(network) + + # Sort both for consistent comparison + sort_cols = ["generator", "investment_period", "timestep"] + result = result.sort_values(sort_cols).reset_index(drop=True) + expected_df = expected_df.sort_values(sort_cols).reset_index(drop=True) + + pd.testing.assert_frame_equal(result, expected_df) + + def test_extract_demand(csv_str_to_df): """Test extraction of demand/load results.""" @@ -138,9 +203,10 @@ def test_extract_generation_expansion_results(csv_str_to_df): """ generators = csv_str_to_df(generators_csv).set_index("Generator") - # 2. Mock Network + # 2. Mock Network with empty storage units network = Mock() network.generators = generators + network.storage_units = pd.DataFrame() # Empty storage units # 3. Expected Output # - custom_gen should be removed @@ -163,3 +229,46 @@ def test_extract_generation_expansion_results(csv_str_to_df): expected_df = expected_df.sort_values(sort_cols).reset_index(drop=True) pd.testing.assert_frame_equal(result, expected_df) + + +def test_extract_generation_expansion_results_with_storage(csv_str_to_df): + """Test extraction of generation expansion results including storage units.""" + + # 1. Setup Generator Data + generators_csv = """ + Generator, bus, carrier, isp_technology_type, build_year, lifetime, p_nom_opt + gen1, NSW1, Gas, Gas - CCGT, 2025, 10, 500.0 + """ + generators = csv_str_to_df(generators_csv).set_index("Generator") + + # 2. Setup Storage Unit Data + storage_csv = """ + StorageUnit, bus, carrier, build_year, lifetime, p_nom_opt + battery1, NSW1, Battery, 2030, 20, 100.0 + battery2, VIC1, Battery, 2035, 15, 200.0 + """ + storage_units = csv_str_to_df(storage_csv).set_index("StorageUnit") + + # 3. Mock Network + network = Mock() + network.generators = generators + network.storage_units = storage_units + + # 4. Expected Output - generators and storage combined + expected_csv = """ + generator, fuel_type, technology_type, node, capacity_mw, investment_period, closure_year + gen1, Gas, Gas - CCGT, NSW1, 500.0, 2025, 2035 + battery1, Battery, Battery Storage, NSW1, 100.0, 2030, 2050 + battery2, Battery, Battery Storage, VIC1, 200.0, 2035, 2050 + """ + expected_df = csv_str_to_df(expected_csv) + + # 5. Run Function + result = extract_generation_expansion_results(network) + + # Sort both for consistent comparison + sort_cols = ["generator"] + result = result.sort_values(sort_cols).reset_index(drop=True) + expected_df = expected_df.sort_values(sort_cols).reset_index(drop=True) + + pd.testing.assert_frame_equal(result, expected_df) From f9b798ee0ee4f5185c0bd2b958c815c3f693aecd Mon Sep 17 00:00:00 2001 From: nick-gorman <40549624+nick-gorman@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:39:02 +1100 Subject: [PATCH 11/12] readme link docs, release v0.1.0beta1, shorter model run --- README.md | 12 ++++++------ ispypsa_config.yaml | 2 +- pyproject.toml | 5 ++--- uv.lock | 12 ++++++------ 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f5dc539e..a55de4cb 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ An open-source capacity expansion modelling tool based on the methodology and assumptions used by the Australian Energy Market Operator (AEMO) to produce their Integrated System Plan (ISP). Built on [PyPSA](https://github.com/pypsa/pypsa). -**This README is a quick reference.** For detailed instructions, tutorials, and API documentation, see the [full documentation](docs/) (hosted docs coming soon): +**This README is a quick reference.** For detailed instructions, tutorials, and API documentation, see the [full documentation](https://open-isp.github.io/ISPyPSA/): -- [Getting Started](docs/getting_started.md) - Installation and first model run -- [Configuration Reference](docs/config.md) - All configuration options -- [CLI Guide](docs/cli.md) - Command line interface details -- [API Reference](docs/api.md) - Python API for custom workflows -- [Workflow Overview](docs/workflow.md) - How the modelling pipeline works +- [Getting Started](https://open-isp.github.io/ISPyPSA/getting_started/) - Installation and first model run +- [Configuration Reference](https://open-isp.github.io/ISPyPSA/config/) - All configuration options +- [CLI Guide](https://open-isp.github.io/ISPyPSA/cli/) - Command line interface details +- [API Reference](https://open-isp.github.io/ISPyPSA/api/) - Python API for custom workflows +- [Workflow Overview](https://open-isp.github.io/ISPyPSA/workflow/) - How the modelling pipeline works ## Installation diff --git a/ispypsa_config.yaml b/ispypsa_config.yaml index aeaf3044..ec914544 100644 --- a/ispypsa_config.yaml +++ b/ispypsa_config.yaml @@ -122,7 +122,7 @@ temporal: # ~ (None): Full yearly temporal representation is used or another aggregation. # list[str]: peak-demand, residual-peak-demand, minimum-demand, # residual-minimum-demand, peak-consumption, residual-peak-consumption - named_representative_weeks: [residual-peak-demand, peak-consumption, residual-minimum-demand] + named_representative_weeks: [residual-peak-demand] operational: resolution_min: 30 diff --git a/pyproject.toml b/pyproject.toml index f5491601..f6be1ae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ISPyPSA" -version = "0.1.0" +version = "0.1.0beta1" description = "An open-source capacity expansion model based on the methodology and datasets used by the Australian Energy Market Operator (AEMO) in their Integrated System Plan (ISP)." authors = [ { name = "prakaa", email = "abiprakash007@gmail.com" }, @@ -14,10 +14,9 @@ dependencies = [ "doit>=0.36.0", "xmltodict>=0.13.0", "thefuzz>=0.22.1", - "isp-trace-parser>=1.0.3", "pyarrow>=18.0.0", "tables>=3.10.1", - "isp-trace-parser>=2.0.2", + "isp-trace-parser>=2.0.3", "isp-workbook-parser>=2.6.0", "requests>=2.32.3", "tqdm>=4.67.1", diff --git a/uv.lock b/uv.lock index c3c96570..671206d0 100644 --- a/uv.lock +++ b/uv.lock @@ -1463,21 +1463,22 @@ wheels = [ [[package]] name = "isp-trace-parser" -version = "2.0.2" +version = "2.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "duckdb" }, { name = "joblib" }, { name = "pandas" }, { name = "polars" }, + { name = "pyarrow" }, { name = "pydantic" }, { name = "pyyaml" }, { name = "requests" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e0/9c/db0183e3d3f0da359f49e177c424784d6067d56c029d10aa01655cf737db/isp_trace_parser-2.0.2.tar.gz", hash = "sha256:7b4c863b66d87ea658e31fbb3474f81dd28edb39adb8d4b7fe3ab7882097918e", size = 57856 } +sdist = { url = "https://files.pythonhosted.org/packages/96/6a/7ceffb77448a7115bfdffe34fc4e70630dab2f09eda88d0c2c370048eb1c/isp_trace_parser-2.0.3.tar.gz", hash = "sha256:cfe3b6b2243c2302a5393a7797c75d91b74b5655b8b947af782860a4e48a9e96", size = 57882 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/e5/478358761c77a2ead9a3b362c751d933f199a49a6534e852ed729bcda718/isp_trace_parser-2.0.2-py3-none-any.whl", hash = "sha256:ea095c843e08bdb1c2aa68c08a3189854598da96befcca75028cc6a0bcdee56d", size = 53312 }, + { url = "https://files.pythonhosted.org/packages/c0/63/598df55262660963716dfb131feb7925ebb89525e5355738c6739d51892a/isp_trace_parser-2.0.3-py3-none-any.whl", hash = "sha256:79b01f72370ee4408f0f0155a538047f6c76e4be373af2a32ad8fa260a844098", size = 53325 }, ] [[package]] @@ -1498,7 +1499,7 @@ wheels = [ [[package]] name = "ispypsa" -version = "0.1.0" +version = "0.1.0b1" source = { editable = "." } dependencies = [ { name = "doit" }, @@ -1552,8 +1553,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "doit", specifier = ">=0.36.0" }, - { name = "isp-trace-parser", specifier = ">=1.0.3" }, - { name = "isp-trace-parser", specifier = ">=2.0.2" }, + { name = "isp-trace-parser", specifier = ">=2.0.3" }, { name = "isp-workbook-parser", specifier = ">=2.6.0" }, { name = "linopy", marker = "extra == 'solvers'", specifier = ">=0.4.4" }, { name = "mkdocs", specifier = ">=1.6.1" }, From fb9841e93f761a970f8e25ada269d8a9e34360a1 Mon Sep 17 00:00:00 2001 From: nick-gorman <40549624+nick-gorman@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:53:23 +1100 Subject: [PATCH 12/12] battery data in plotting tests --- tests/test_plotting/test_generation.py | 27 ++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_plotting/test_generation.py b/tests/test_plotting/test_generation.py index 9b3a328d..884df71a 100644 --- a/tests/test_plotting/test_generation.py +++ b/tests/test_plotting/test_generation.py @@ -178,17 +178,29 @@ def test_plot_dispatch_with_geography(csv_str_to_df): def test_plot_dispatch_system_level(csv_str_to_df): - """Test plot_dispatch without geography_level returns system-level structure.""" + """Test plot_dispatch without geography_level returns system-level structure. + + Includes battery data with both charging (negative) and discharging (positive) + to verify battery traces are correctly split in the plot. Battery charging + and discharging are on separate timesteps to ensure both traces are created + (since values are aggregated by timestep before splitting). + """ dispatch_csv = """ generator, node, fuel_type, investment_period, timestep, dispatch_mw Gen1, SubA, Coal, 2024, 2024-01-01 12:00:00, 100 Gen2, SubB, Wind, 2024, 2024-01-01 12:00:00, 50 + Bat1, SubA, Battery, 2024, 2024-01-01 12:00:00, 30 + Gen1, SubA, Coal, 2024, 2024-01-01 13:00:00, 100 + Gen2, SubB, Wind, 2024, 2024-01-01 13:00:00, 50 + Bat1, SubA, Battery, 2024, 2024-01-01 13:00:00, -20 """ demand_csv = """ node, load, investment_period, timestep, demand_mw SubA, Load1, 2024, 2024-01-01 12:00:00, 80 SubB, Load2, 2024, 2024-01-01 12:00:00, 70 + SubA, Load1, 2024, 2024-01-01 13:00:00, 80 + SubB, Load2, 2024, 2024-01-01 13:00:00, 70 """ dispatch = csv_str_to_df(dispatch_csv) @@ -204,9 +216,16 @@ def test_plot_dispatch_system_level(csv_str_to_df): entry = result["2024"]["2024-01-01"] assert isinstance(entry["plot"], go.Figure) assert isinstance(entry["data"], pd.DataFrame) - # Total dispatch should be 150 (100 + 50) - assert entry["data"]["dispatch_mw"].sum() == 150 - assert set(entry["data"]["fuel_type"].unique()) == {"Coal", "Wind"} + # Total dispatch should be 310 (100+50+30 + 100+50-20) + assert entry["data"]["dispatch_mw"].sum() == 310 + assert set(entry["data"]["fuel_type"].unique()) == {"Coal", "Wind", "Battery"} + + # Check that plot has separate battery charging and discharging traces + trace_names = [trace.name for trace in entry["plot"].data] + assert "Battery Discharging" in trace_names + assert "Battery Charging" in trace_names + # Raw "Battery" should not appear as a trace (it's split) + assert "Battery" not in trace_names def test_prepare_generation_capacity(csv_str_to_df):