From 13d0ce157de5a68e7ea4403f27c5e923250ec819 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Tue, 8 Jul 2025 11:57:26 +0200 Subject: [PATCH 1/2] use queue and listener for logging --- packages/helpermodules/logger.py | 67 +++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index e164861394..f28cf984ae 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -1,7 +1,9 @@ import functools import logging +import logging.handlers from logging.handlers import RotatingFileHandler from pathlib import Path +import queue import sys import threading import typing_extensions @@ -106,72 +108,117 @@ def filter_pos(name: str, record) -> bool: def setup_logging() -> None: def mb_to_bytes(megabytes: int) -> int: return megabytes * 1000000 - # Mehrere kleine Dateien verwenden, damit nicht zu viel verworfen wird, wenn die Datei voll ist. + + # Main logger + log_queue = queue.Queue() + queue_handler = logging.handlers.QueueHandler(log_queue) main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5.5), backupCount=4) main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) main_file_handler.addFilter(RedactingFilter()) - logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler]) + logging.basicConfig(level=logging.DEBUG, handlers=[queue_handler]) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome")) + main_listener = logging.handlers.QueueListener(log_queue, main_file_handler) + main_listener.start() + # Chargelog logger + chargelog_queue = queue.Queue() + chargelog_queue_handler = logging.handlers.QueueHandler(chargelog_queue) chargelog_log = logging.getLogger("chargelog") chargelog_log.propagate = False chargelog_file_handler = RotatingFileHandler( RAMDISK_PATH + 'chargelog.log', maxBytes=mb_to_bytes(2), backupCount=1) chargelog_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) chargelog_file_handler.addFilter(RedactingFilter()) - chargelog_log.addHandler(chargelog_file_handler) + chargelog_log.addHandler(chargelog_queue_handler) + chargelog_listener = logging.handlers.QueueListener(chargelog_queue, chargelog_file_handler) + chargelog_listener.start() + # Data migration logger + data_migration_queue = queue.Queue() + data_migration_queue_handler = logging.handlers.QueueHandler(data_migration_queue) data_migration_log = logging.getLogger("data_migration") data_migration_log.propagate = False data_migration_file_handler = RotatingFileHandler( PERSISTENT_LOG_PATH + 'data_migration.log', maxBytes=mb_to_bytes(1), backupCount=1) data_migration_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) data_migration_file_handler.addFilter(RedactingFilter()) - data_migration_log.addHandler(data_migration_file_handler) + data_migration_log.addHandler(data_migration_queue_handler) + data_migration_listener = logging.handlers.QueueListener(data_migration_queue, data_migration_file_handler) + data_migration_listener.start() + # MQTT logger + mqtt_queue = queue.Queue() + mqtt_queue_handler = logging.handlers.QueueHandler(mqtt_queue) mqtt_log = logging.getLogger("mqtt") mqtt_log.propagate = False mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1) mqtt_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) mqtt_file_handler.addFilter(RedactingFilter()) - mqtt_log.addHandler(mqtt_file_handler) + mqtt_log.addHandler(mqtt_queue_handler) + mqtt_listener = logging.handlers.QueueListener(mqtt_queue, mqtt_file_handler) + mqtt_listener.start() + # Steuve control command logger + steuve_control_command_queue = queue.Queue() + steuve_control_command_queue_handler = logging.handlers.QueueHandler(steuve_control_command_queue) steuve_control_command_log = logging.getLogger("steuve_control_command") steuve_control_command_log.propagate = False steuve_control_command_file_handler = RotatingFileHandler( PERSISTENT_LOG_PATH + 'steuve_control_command.log', maxBytes=mb_to_bytes(80), backupCount=1) steuve_control_command_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) - steuve_control_command_log.addHandler(steuve_control_command_file_handler) + steuve_control_command_log.addHandler(steuve_control_command_queue_handler) + steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue, steuve_control_command_file_handler) + steuve_control_command_listener.start() + # Smarthome logger + smarthome_queue = queue.Queue() + smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue) smarthome_log_handler = RotatingFileHandler(RAMDISK_PATH + 'smarthome.log', maxBytes=mb_to_bytes(1), backupCount=1) smarthome_log_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) smarthome_log_handler.addFilter(functools.partial(filter_pos, "smarthome")) smarthome_log_handler.addFilter(RedactingFilter()) - logging.getLogger().addHandler(smarthome_log_handler) + logging.getLogger().addHandler(smarthome_queue_handler) + smarthome_listener = logging.handlers.QueueListener(smarthome_queue, smarthome_log_handler) + smarthome_listener.start() + # SoC logger + soc_queue = queue.Queue() + soc_queue_handler = logging.handlers.QueueHandler(soc_queue) soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) soc_log_handler.addFilter(functools.partial(filter_pos, "soc")) soc_log_handler.addFilter(RedactingFilter()) - logging.getLogger().addHandler(soc_log_handler) + logging.getLogger().addHandler(soc_queue_handler) + soc_listener = logging.handlers.QueueListener(soc_queue, soc_log_handler) + soc_listener.start() + # Internal chargepoint logger + internal_chargepoint_queue = queue.Queue() + internal_chargepoint_queue_handler = logging.handlers.QueueHandler(internal_chargepoint_queue) internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log', maxBytes=mb_to_bytes(1), backupCount=1) internal_chargepoint_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint")) internal_chargepoint_log_handler.addFilter(RedactingFilter()) - logging.getLogger().addHandler(internal_chargepoint_log_handler) + logging.getLogger().addHandler(internal_chargepoint_queue_handler) + internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue, internal_chargepoint_log_handler) + internal_chargepoint_listener.start() + # urllib3 logger + urllib3_queue = queue.Queue() + urllib3_queue_handler = logging.handlers.QueueHandler(urllib3_queue) urllib3_log = logging.getLogger("urllib3.connectionpool") urllib3_log.propagate = True urllib3_file_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) urllib3_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) urllib3_file_handler.addFilter(RedactingFilter()) urllib3_file_handler.addFilter(functools.partial(filter_pos, "soc")) - urllib3_log.addHandler(urllib3_file_handler) + urllib3_log.addHandler(urllib3_queue_handler) + urllib3_listener = logging.handlers.QueueListener(urllib3_queue, urllib3_file_handler) + urllib3_listener.start() logging.getLogger("pymodbus").setLevel(logging.WARNING) logging.getLogger("uModbus").setLevel(logging.WARNING) From 7b1f39646d91e1a31d2f3f7a098263f6d56ee32c Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 9 Jul 2025 09:16:39 +0200 Subject: [PATCH 2/2] flake8 --- packages/helpermodules/logger.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index f28cf984ae..016305c5f1 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -169,7 +169,8 @@ def mb_to_bytes(megabytes: int) -> int: PERSISTENT_LOG_PATH + 'steuve_control_command.log', maxBytes=mb_to_bytes(80), backupCount=1) steuve_control_command_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) steuve_control_command_log.addHandler(steuve_control_command_queue_handler) - steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue, steuve_control_command_file_handler) + steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue, + steuve_control_command_file_handler) steuve_control_command_listener.start() # Smarthome logger @@ -204,7 +205,8 @@ def mb_to_bytes(megabytes: int) -> int: internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint")) internal_chargepoint_log_handler.addFilter(RedactingFilter()) logging.getLogger().addHandler(internal_chargepoint_queue_handler) - internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue, internal_chargepoint_log_handler) + internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue, + internal_chargepoint_log_handler) internal_chargepoint_listener.start() # urllib3 logger