From 7274e149767ba324349c0ba319a3e5ac01ecb7d4 Mon Sep 17 00:00:00 2001 From: AbyAbraham21 <95077500+AbyAbraham21@users.noreply.github.com> Date: Wed, 22 Jan 2025 19:59:04 +0000 Subject: [PATCH 1/6] Update eco.py Changes to the CodeCarbon API means that the output values will need to be computed in a different manner. Delta emissions will need to be derived if needed. --- simvue/eco.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/simvue/eco.py b/simvue/eco.py index 6ff7023b..4dd7f9ba 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -34,10 +34,10 @@ def out( logger.debug("Logging CodeCarbon metadata") self._simvue_run.update_metadata( { - "codecarbon.country": total.country_name, - "codecarbon.country_iso_code": total.country_iso_code, - "codecarbon.region": total.region, - "codecarbon.version": total.codecarbon_version, + "codecarbon.country": total.final_emissions_data.country_name, + "codecarbon.country_iso_code": total.final_emissions_data.country_iso_code, + "codecarbon.region": total.final_emissions_data.region, + "codecarbon.version": total.final_emissions_data.codecarbon_version, } ) @@ -48,10 +48,8 @@ def out( logger.debug("Logging CodeCarbon metrics") self._simvue_run.log_metrics( metrics={ - "codecarbon.total.emissions": total.emissions, - "codecarbon.total.energy_consumed": total.energy_consumed, - "codecarbon.delta.emissions": delta.emissions, - "codecarbon.delta.energy_consumed": delta.energy_consumed, + "codecarbon.total.emissions": total.final_emissions_data.emissions, + "codecarbon.total.energy_consumed": total.final_emissions_data.energy_consumed, }, step=self._metrics_step, timestamp=simvue_timestamp(_cc_timestamp), From 1827fedf3721f8c64aa0ce4061067dd40e224a44 Mon Sep 17 00:00:00 2001 From: AbyAbraham21 Date: Thu, 27 Feb 2025 10:22:36 +0000 Subject: [PATCH 2/6] Adding Code carbon changes --- pyproject.toml | 2 +- simvue/eco.py | 68 +++++++++++++++++++++--------- tests/functional/test_run_class.py | 18 ++++---- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6837df44..785daf76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ gitpython = "^3.1.43" humanfriendly = "^10.0" tabulate = "^0.9.0" randomname = "^0.2.1" -codecarbon = "^2.7.1" +codecarbon = "^2.8.1" numpy = "^2.1.2" flatdict = "^4.0.1" semver = "^3.0.2" diff --git a/simvue/eco.py b/simvue/eco.py index 4dd7f9ba..a9050848 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -1,9 +1,9 @@ import typing import logging import datetime - + from codecarbon import EmissionsTracker -from codecarbon.output_methods.base_output import BaseOutput as cc_BaseOutput +from codecarbon.output import BaseOutput as cc_BaseOutput from simvue.utilities import simvue_timestamp if typing.TYPE_CHECKING: @@ -18,6 +18,8 @@ class CodeCarbonOutput(cc_BaseOutput): def __init__(self, run: "Run") -> None: self._simvue_run = run self._metrics_step: int = 0 + self.emissions = 0.0 # To store the CO2 emissions data + self.energy_consumed = 0.0 # To store the energy consumed data def out( self, total: "EmissionsData", delta: "EmissionsData", meta_update: bool = True @@ -32,33 +34,59 @@ def out( if meta_update: logger.debug("Logging CodeCarbon metadata") - self._simvue_run.update_metadata( - { - "codecarbon.country": total.final_emissions_data.country_name, - "codecarbon.country_iso_code": total.final_emissions_data.country_iso_code, - "codecarbon.region": total.final_emissions_data.region, - "codecarbon.version": total.final_emissions_data.codecarbon_version, - } + try: + self._simvue_run.update_metadata( + { + "codecarbon.country": total.country_name, + "codecarbon.country_iso_code": total.country_iso_code, + "codecarbon.region": total.region, + "codecarbon.version": total.codecarbon_version, + } + ) + except AttributeError as e: + logger.error(f"Failed to update metadata: {e}") + try: + _cc_timestamp = datetime.datetime.strptime( + total.timestamp, "%Y-%m-%dT%H:%M:%S" ) + except ValueError as e: + logger.error(f"Error parsing timestamp: {e}") + return - _cc_timestamp: datetime.datetime = datetime.datetime.strptime( - total.timestamp, "%Y-%m-%dT%H:%M:%S" - ) + # Accumulate the emissions and energy consumed + self.emissions += total.emissions # Add new emissions to the total + self.energy_consumed += total.energy_consumed # Add new energy consumed to the total logger.debug("Logging CodeCarbon metrics") - self._simvue_run.log_metrics( - metrics={ - "codecarbon.total.emissions": total.final_emissions_data.emissions, - "codecarbon.total.energy_consumed": total.final_emissions_data.energy_consumed, - }, - step=self._metrics_step, - timestamp=simvue_timestamp(_cc_timestamp), - ) + print("total.emissions=", self.emissions) + print("total.energy_consumed=", self.energy_consumed) + print("total.timestamp=",total.timestamp) + print("_cc_timestamp=",_cc_timestamp) + try: + self._simvue_run.log_metrics( + metrics={ + "codecarbon.emissions": total.emissions, + "codecarbon.energy_consumed": total.energy_consumed, + }, + step=self._metrics_step, + timestamp=simvue_timestamp(_cc_timestamp), + ) + except ArithmeticError as e: + logger.error(f"Failed to log metrics: {e}") + return + self._metrics_step += 1 def live_out(self, total: "EmissionsData", delta: "EmissionsData") -> None: self.out(total, delta, meta_update=False) + def get_total_emissions(self) -> float: + """Getter for the total accumulated emissions""" + return self.emissions + + def get_total_energy_consumed(self) -> float: + """Getter for the total accumulated energy consumed""" + return self.energy_consumed class SimvueEmissionsTracker(EmissionsTracker): def __init__( diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 304fbad9..8ef02102 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -47,15 +47,15 @@ def test_check_run_initialised_decorator() -> None: assert "Simvue Run must be initialised" in str(e.value) -# @pytest.mark.run -# def test_run_with_emissions() -> None: -# with sv_run.Run() as run_created: -# run_created.init(retention_period="1 min") -# run_created.config(enable_emission_metrics=True, emission_metrics_interval=1) -# time.sleep(5) -# _run = RunObject(identifier=run_created.id) -# import pdb; pdb.set_trace() -# assert list(_run.metrics) +@pytest.mark.run +def test_run_with_emissions() -> None: + with sv_run.Run() as run_created: + run_created.init(retention_period="1 min") + run_created.config(enable_emission_metrics=True, emission_metrics_interval=1) + time.sleep(5) + _run = RunObject(identifier=run_created.id) + import pdb; pdb.set_trace() + assert list(_run.metrics) @pytest.mark.run From ff603c4d11551af9a5666d3ae44210f95a627777 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Thu, 27 Feb 2025 10:48:02 +0000 Subject: [PATCH 3/6] Add nested metadata dict --- simvue/eco.py | 23 ++++++++++++----------- tests/functional/test_run_class.py | 6 ++++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/simvue/eco.py b/simvue/eco.py index a9050848..20ea124e 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -1,7 +1,7 @@ import typing import logging import datetime - + from codecarbon import EmissionsTracker from codecarbon.output import BaseOutput as cc_BaseOutput from simvue.utilities import simvue_timestamp @@ -37,10 +37,12 @@ def out( try: self._simvue_run.update_metadata( { - "codecarbon.country": total.country_name, - "codecarbon.country_iso_code": total.country_iso_code, - "codecarbon.region": total.region, - "codecarbon.version": total.codecarbon_version, + "codecarbon": { + "country": total.country_name, + "country_iso_code": total.country_iso_code, + "region": total.region, + "version": total.codecarbon_version, + } } ) except AttributeError as e: @@ -55,13 +57,11 @@ def out( # Accumulate the emissions and energy consumed self.emissions += total.emissions # Add new emissions to the total - self.energy_consumed += total.energy_consumed # Add new energy consumed to the total + self.energy_consumed += ( + total.energy_consumed + ) # Add new energy consumed to the total logger.debug("Logging CodeCarbon metrics") - print("total.emissions=", self.emissions) - print("total.energy_consumed=", self.energy_consumed) - print("total.timestamp=",total.timestamp) - print("_cc_timestamp=",_cc_timestamp) try: self._simvue_run.log_metrics( metrics={ @@ -74,7 +74,7 @@ def out( except ArithmeticError as e: logger.error(f"Failed to log metrics: {e}") return - + self._metrics_step += 1 def live_out(self, total: "EmissionsData", delta: "EmissionsData") -> None: @@ -88,6 +88,7 @@ def get_total_energy_consumed(self) -> float: """Getter for the total accumulated energy consumed""" return self.energy_consumed + class SimvueEmissionsTracker(EmissionsTracker): def __init__( self, project_name: str, simvue_run: "Run", metrics_interval: int diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index e3654e06..bb615de9 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -53,9 +53,11 @@ def test_run_with_emissions() -> None: with sv_run.Run() as run_created: run_created.init(retention_period="1 min") run_created.config(enable_emission_metrics=True, emission_metrics_interval=1) - time.sleep(5) + time.sleep(60) _run = RunObject(identifier=run_created.id) - assert list(_run.metrics) + _metric_names = [item[0] for item in _run.metrics] + assert 'codecarbon.energy_consumed' in _metric_names + assert 'codecarbon.emissions' in _metric_names @pytest.mark.run From ec5052559668cf0fca124bf380537a50ae6d6a34 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 28 Feb 2025 09:33:07 +0000 Subject: [PATCH 4/6] Improved test, fixed api call time --- simvue/eco.py | 13 +++++-------- tests/functional/test_run_class.py | 21 ++++++++++++++++----- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/simvue/eco.py b/simvue/eco.py index 20ea124e..b874923a 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -55,18 +55,14 @@ def out( logger.error(f"Error parsing timestamp: {e}") return - # Accumulate the emissions and energy consumed - self.emissions += total.emissions # Add new emissions to the total - self.energy_consumed += ( - total.energy_consumed - ) # Add new energy consumed to the total - logger.debug("Logging CodeCarbon metrics") try: self._simvue_run.log_metrics( metrics={ - "codecarbon.emissions": total.emissions, - "codecarbon.energy_consumed": total.energy_consumed, + "codecarbon.total.emissions": total.emissions, + "codecarbon.total.energy_consumed": total.energy_consumed, + "codecarbon.delta.emissions": delta.emissions, + "codecarbon.delta.energy_consumed": delta.energy_consumed, }, step=self._metrics_step, timestamp=simvue_timestamp(_cc_timestamp), @@ -98,6 +94,7 @@ def __init__( super().__init__( project_name=project_name, measure_power_secs=metrics_interval, + api_call_interval=1, experiment_id=None, experiment_name=None, logging_logger=CodeCarbonOutput(simvue_run), diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index bb615de9..e4f6f55a 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -53,12 +53,23 @@ def test_run_with_emissions() -> None: with sv_run.Run() as run_created: run_created.init(retention_period="1 min") run_created.config(enable_emission_metrics=True, emission_metrics_interval=1) - time.sleep(60) + time.sleep(5) _run = RunObject(identifier=run_created.id) - _metric_names = [item[0] for item in _run.metrics] - assert 'codecarbon.energy_consumed' in _metric_names - assert 'codecarbon.emissions' in _metric_names - + _metric_names = [item[0] for item in _run.metrics] + client = sv_cl.Client() + for _metric in ["emissions", "energy_consumed"]: + _total_metric_name = f'codecarbon.total.{_metric}' + _delta_metric_name = f'codecarbon.delta.{_metric}' + assert _total_metric_name in _metric_names + assert _delta_metric_name in _metric_names + _metric_values = client.get_metric_values(metric_names=[_total_metric_name, _delta_metric_name], xaxis="time", output_format="dataframe", run_ids=[run_created.id]) + + # Check that total = previous total + latest delta + _total_values = _metric_values[_total_metric_name].tolist() + _delta_values = _metric_values[_delta_metric_name].tolist() + assert len(_total_values) > 1 + for i in range(1, len(_total_values)): + assert _total_values[i] == _total_values[i-1] + _delta_values[i] @pytest.mark.run @pytest.mark.parametrize("timestamp", (datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f"), None), ids=("timestamp", "no_timestamp")) From a560a8241817b8f8e8c1881818cae0c034b9f4e4 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 28 Feb 2025 09:39:15 +0000 Subject: [PATCH 5/6] Remove unused attributes --- simvue/eco.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/simvue/eco.py b/simvue/eco.py index b874923a..42caa528 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -18,8 +18,6 @@ class CodeCarbonOutput(cc_BaseOutput): def __init__(self, run: "Run") -> None: self._simvue_run = run self._metrics_step: int = 0 - self.emissions = 0.0 # To store the CO2 emissions data - self.energy_consumed = 0.0 # To store the energy consumed data def out( self, total: "EmissionsData", delta: "EmissionsData", meta_update: bool = True @@ -76,14 +74,6 @@ def out( def live_out(self, total: "EmissionsData", delta: "EmissionsData") -> None: self.out(total, delta, meta_update=False) - def get_total_emissions(self) -> float: - """Getter for the total accumulated emissions""" - return self.emissions - - def get_total_energy_consumed(self) -> float: - """Getter for the total accumulated energy consumed""" - return self.energy_consumed - class SimvueEmissionsTracker(EmissionsTracker): def __init__( From bae99c44f9d3adcce63fa426bfaee4958a5a2793 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 28 Feb 2025 11:38:06 +0000 Subject: [PATCH 6/6] Added support for codecarbon in offline mode --- simvue/config/parameters.py | 1 + simvue/eco.py | 31 +++++++++++++++++++++++- simvue/run.py | 48 ++++++++++++++++++++++++++++++------- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/simvue/config/parameters.py b/simvue/config/parameters.py index c6d65c93..9e0b38bc 100644 --- a/simvue/config/parameters.py +++ b/simvue/config/parameters.py @@ -48,6 +48,7 @@ def check_token(cls, v: typing.Any) -> str | None: class OfflineSpecifications(pydantic.BaseModel): cache: pathlib.Path | None = None + country_iso_code: str | None = None class MetricsSpecifications(pydantic.BaseModel): diff --git a/simvue/eco.py b/simvue/eco.py index 42caa528..50db508f 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -2,7 +2,7 @@ import logging import datetime -from codecarbon import EmissionsTracker +from codecarbon import EmissionsTracker, OfflineEmissionsTracker from codecarbon.output import BaseOutput as cc_BaseOutput from simvue.utilities import simvue_timestamp @@ -101,3 +101,32 @@ def post_init(self) -> None: self._set_from_conf(self._simvue_run._id, "experiment_id") self._set_from_conf(self._simvue_run._name, "experiment_name") self.start() + + +class OfflineSimvueEmissionsTracker(OfflineEmissionsTracker): + def __init__( + self, project_name: str, simvue_run: "Run", metrics_interval: int + ) -> None: + self._simvue_run = simvue_run + logger.setLevel(logging.ERROR) + super().__init__( + country_iso_code=simvue_run._user_config.offline.country_iso_code, + project_name=project_name, + measure_power_secs=metrics_interval, + api_call_interval=1, + experiment_id=None, + experiment_name=None, + logging_logger=CodeCarbonOutput(simvue_run), + save_to_logger=True, + allow_multiple_runs=True, + log_level="error", + ) + + def set_measure_interval(self, interval: int) -> None: + """Set the measure interval""" + self._set_from_conf(interval, "measure_power_secs") + + def post_init(self) -> None: + self._set_from_conf(self._simvue_run._id, "experiment_id") + self._set_from_conf(self._simvue_run._name, "experiment_name") + self.start() diff --git a/simvue/run.py b/simvue/run.py index ec574013..c2448026 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -43,7 +43,7 @@ from .models import FOLDER_REGEX, NAME_REGEX, MetricKeyString from .system import get_system from .metadata import git_info, environment -from .eco import SimvueEmissionsTracker +from .eco import SimvueEmissionsTracker, OfflineSimvueEmissionsTracker from .utilities import ( skip_if_failed, validate_timestamp, @@ -208,11 +208,28 @@ def __init__( ) else self._user_config.metrics.emission_metrics_interval ) - self._emissions_tracker: SimvueEmissionsTracker | None = ( - SimvueEmissionsTracker("simvue", self, self._emission_metrics_interval) - if self._user_config.metrics.enable_emission_metrics - else None - ) + if mode == "offline": + if ( + self._user_config.metrics.enable_emission_metrics + and not self._user_config.offline.country_iso_code + ): + raise ValueError( + "Country ISO code must be provided if tracking emissions metrics in offline mode." + ) + + self._emissions_tracker: OfflineSimvueEmissionsTracker | None = ( + OfflineSimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + if self._user_config.metrics.enable_emission_metrics + else None + ) + else: + self._emissions_tracker: SimvueEmissionsTracker | None = ( + SimvueEmissionsTracker("simvue", self, self._emission_metrics_interval) + if self._user_config.metrics.enable_emission_metrics + else None + ) def __enter__(self) -> Self: return self @@ -1028,9 +1045,22 @@ def config( self._emission_metrics_interval = emission_metrics_interval if enable_emission_metrics: - self._emissions_tracker = SimvueEmissionsTracker( - "simvue", self, self._emission_metrics_interval - ) + if self._user_config.run.mode == "offline": + if not self._user_config.offline.country_iso_code: + self._error( + "Country ISO code must be provided if tracking emissions metrics in offline mode." + ) + self._emissions_tracker: OfflineSimvueEmissionsTracker = ( + OfflineSimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + ) + else: + self._emissions_tracker: SimvueEmissionsTracker = ( + SimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + ) # If the main Run API object is initialised the run is active # hence the tracker should start too