diff --git a/simvue/config/parameters.py b/simvue/config/parameters.py index c6d65c93..9e0b38bc 100644 --- a/simvue/config/parameters.py +++ b/simvue/config/parameters.py @@ -48,6 +48,7 @@ def check_token(cls, v: typing.Any) -> str | None: class OfflineSpecifications(pydantic.BaseModel): cache: pathlib.Path | None = None + country_iso_code: str | None = None class MetricsSpecifications(pydantic.BaseModel): diff --git a/simvue/eco.py b/simvue/eco.py index 6ff7023b..50db508f 100644 --- a/simvue/eco.py +++ b/simvue/eco.py @@ -2,8 +2,8 @@ import logging import datetime -from codecarbon import EmissionsTracker -from codecarbon.output_methods.base_output import BaseOutput as cc_BaseOutput +from codecarbon import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.output import BaseOutput as cc_BaseOutput from simvue.utilities import simvue_timestamp if typing.TYPE_CHECKING: @@ -32,30 +32,43 @@ def out( if meta_update: logger.debug("Logging CodeCarbon metadata") - self._simvue_run.update_metadata( - { - "codecarbon.country": total.country_name, - "codecarbon.country_iso_code": total.country_iso_code, - "codecarbon.region": total.region, - "codecarbon.version": total.codecarbon_version, - } + try: + self._simvue_run.update_metadata( + { + "codecarbon": { + "country": total.country_name, + "country_iso_code": total.country_iso_code, + "region": total.region, + "version": total.codecarbon_version, + } + } + ) + except AttributeError as e: + logger.error(f"Failed to update metadata: {e}") + try: + _cc_timestamp = datetime.datetime.strptime( + total.timestamp, "%Y-%m-%dT%H:%M:%S" ) - - _cc_timestamp: datetime.datetime = datetime.datetime.strptime( - total.timestamp, "%Y-%m-%dT%H:%M:%S" - ) + except ValueError as e: + logger.error(f"Error parsing timestamp: {e}") + return logger.debug("Logging CodeCarbon metrics") - self._simvue_run.log_metrics( - metrics={ - "codecarbon.total.emissions": total.emissions, - "codecarbon.total.energy_consumed": total.energy_consumed, - "codecarbon.delta.emissions": delta.emissions, - "codecarbon.delta.energy_consumed": delta.energy_consumed, - }, - step=self._metrics_step, - timestamp=simvue_timestamp(_cc_timestamp), - ) + try: + self._simvue_run.log_metrics( + metrics={ + "codecarbon.total.emissions": total.emissions, + "codecarbon.total.energy_consumed": total.energy_consumed, + "codecarbon.delta.emissions": delta.emissions, + "codecarbon.delta.energy_consumed": delta.energy_consumed, + }, + step=self._metrics_step, + timestamp=simvue_timestamp(_cc_timestamp), + ) + except ArithmeticError as e: + logger.error(f"Failed to log metrics: {e}") + return + self._metrics_step += 1 def live_out(self, total: "EmissionsData", delta: "EmissionsData") -> None: @@ -71,6 +84,36 @@ def __init__( super().__init__( project_name=project_name, measure_power_secs=metrics_interval, + api_call_interval=1, + experiment_id=None, + experiment_name=None, + logging_logger=CodeCarbonOutput(simvue_run), + save_to_logger=True, + allow_multiple_runs=True, + log_level="error", + ) + + def set_measure_interval(self, interval: int) -> None: + """Set the measure interval""" + self._set_from_conf(interval, "measure_power_secs") + + def post_init(self) -> None: + self._set_from_conf(self._simvue_run._id, "experiment_id") + self._set_from_conf(self._simvue_run._name, "experiment_name") + self.start() + + +class OfflineSimvueEmissionsTracker(OfflineEmissionsTracker): + def __init__( + self, project_name: str, simvue_run: "Run", metrics_interval: int + ) -> None: + self._simvue_run = simvue_run + logger.setLevel(logging.ERROR) + super().__init__( + country_iso_code=simvue_run._user_config.offline.country_iso_code, + project_name=project_name, + measure_power_secs=metrics_interval, + api_call_interval=1, experiment_id=None, experiment_name=None, logging_logger=CodeCarbonOutput(simvue_run), diff --git a/simvue/run.py b/simvue/run.py index ec574013..c2448026 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -43,7 +43,7 @@ from .models import FOLDER_REGEX, NAME_REGEX, MetricKeyString from .system import get_system from .metadata import git_info, environment -from .eco import SimvueEmissionsTracker +from .eco import SimvueEmissionsTracker, OfflineSimvueEmissionsTracker from .utilities import ( skip_if_failed, validate_timestamp, @@ -208,11 +208,28 @@ def __init__( ) else self._user_config.metrics.emission_metrics_interval ) - self._emissions_tracker: SimvueEmissionsTracker | None = ( - SimvueEmissionsTracker("simvue", self, self._emission_metrics_interval) - if self._user_config.metrics.enable_emission_metrics - else None - ) + if mode == "offline": + if ( + self._user_config.metrics.enable_emission_metrics + and not self._user_config.offline.country_iso_code + ): + raise ValueError( + "Country ISO code must be provided if tracking emissions metrics in offline mode." + ) + + self._emissions_tracker: OfflineSimvueEmissionsTracker | None = ( + OfflineSimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + if self._user_config.metrics.enable_emission_metrics + else None + ) + else: + self._emissions_tracker: SimvueEmissionsTracker | None = ( + SimvueEmissionsTracker("simvue", self, self._emission_metrics_interval) + if self._user_config.metrics.enable_emission_metrics + else None + ) def __enter__(self) -> Self: return self @@ -1028,9 +1045,22 @@ def config( self._emission_metrics_interval = emission_metrics_interval if enable_emission_metrics: - self._emissions_tracker = SimvueEmissionsTracker( - "simvue", self, self._emission_metrics_interval - ) + if self._user_config.run.mode == "offline": + if not self._user_config.offline.country_iso_code: + self._error( + "Country ISO code must be provided if tracking emissions metrics in offline mode." + ) + self._emissions_tracker: OfflineSimvueEmissionsTracker = ( + OfflineSimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + ) + else: + self._emissions_tracker: SimvueEmissionsTracker = ( + SimvueEmissionsTracker( + "simvue", self, self._emission_metrics_interval + ) + ) # If the main Run API object is initialised the run is active # hence the tracker should start too diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index e3654e06..e4f6f55a 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -55,8 +55,21 @@ def test_run_with_emissions() -> None: run_created.config(enable_emission_metrics=True, emission_metrics_interval=1) time.sleep(5) _run = RunObject(identifier=run_created.id) - assert list(_run.metrics) - + _metric_names = [item[0] for item in _run.metrics] + client = sv_cl.Client() + for _metric in ["emissions", "energy_consumed"]: + _total_metric_name = f'codecarbon.total.{_metric}' + _delta_metric_name = f'codecarbon.delta.{_metric}' + assert _total_metric_name in _metric_names + assert _delta_metric_name in _metric_names + _metric_values = client.get_metric_values(metric_names=[_total_metric_name, _delta_metric_name], xaxis="time", output_format="dataframe", run_ids=[run_created.id]) + + # Check that total = previous total + latest delta + _total_values = _metric_values[_total_metric_name].tolist() + _delta_values = _metric_values[_delta_metric_name].tolist() + assert len(_total_values) > 1 + for i in range(1, len(_total_values)): + assert _total_values[i] == _total_values[i-1] + _delta_values[i] @pytest.mark.run @pytest.mark.parametrize("timestamp", (datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f"), None), ids=("timestamp", "no_timestamp"))