Skip to content

use queue and listener for logging #2528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LKuemmel merged 2 commits into from
Jul 9, 2025
Merged

use queue and listener for logging #2528

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
13d0ce1
use queue and listener for logging
LKuemmel Jul 8, 2025
7b1f396
flake8
LKuemmel Jul 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 61 additions & 12 deletions packages/helpermodules/logger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import functools
import logging
import logging.handlers
from logging.handlers import RotatingFileHandler
from pathlib import Path
import queue
import sys
import threading
import typing_extensions
Expand Down Expand Up @@ -106,72 +108,119 @@ def filter_pos(name: str, record) -> bool:
def setup_logging() -> None:
def mb_to_bytes(megabytes: int) -> int:
return megabytes * 1000000
# Mehrere kleine Dateien verwenden, damit nicht zu viel verworfen wird, wenn die Datei voll ist.

# Main logger
log_queue = queue.Queue()
queue_handler = logging.handlers.QueueHandler(log_queue)
main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5.5), backupCount=4)
main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
main_file_handler.addFilter(RedactingFilter())
logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler])
logging.basicConfig(level=logging.DEBUG, handlers=[queue_handler])
logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc"))
logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint"))
logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome"))
main_listener = logging.handlers.QueueListener(log_queue, main_file_handler)
main_listener.start()

# Chargelog logger
chargelog_queue = queue.Queue()
chargelog_queue_handler = logging.handlers.QueueHandler(chargelog_queue)
chargelog_log = logging.getLogger("chargelog")
chargelog_log.propagate = False
chargelog_file_handler = RotatingFileHandler(
RAMDISK_PATH + 'chargelog.log', maxBytes=mb_to_bytes(2), backupCount=1)
chargelog_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
chargelog_file_handler.addFilter(RedactingFilter())
chargelog_log.addHandler(chargelog_file_handler)
chargelog_log.addHandler(chargelog_queue_handler)
chargelog_listener = logging.handlers.QueueListener(chargelog_queue, chargelog_file_handler)
chargelog_listener.start()

# Data migration logger
data_migration_queue = queue.Queue()
data_migration_queue_handler = logging.handlers.QueueHandler(data_migration_queue)
data_migration_log = logging.getLogger("data_migration")
data_migration_log.propagate = False
data_migration_file_handler = RotatingFileHandler(
PERSISTENT_LOG_PATH + 'data_migration.log', maxBytes=mb_to_bytes(1), backupCount=1)
data_migration_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
data_migration_file_handler.addFilter(RedactingFilter())
data_migration_log.addHandler(data_migration_file_handler)
data_migration_log.addHandler(data_migration_queue_handler)
data_migration_listener = logging.handlers.QueueListener(data_migration_queue, data_migration_file_handler)
data_migration_listener.start()

# MQTT logger
mqtt_queue = queue.Queue()
mqtt_queue_handler = logging.handlers.QueueHandler(mqtt_queue)
mqtt_log = logging.getLogger("mqtt")
mqtt_log.propagate = False
mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1)
mqtt_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
mqtt_file_handler.addFilter(RedactingFilter())
mqtt_log.addHandler(mqtt_file_handler)
mqtt_log.addHandler(mqtt_queue_handler)
mqtt_listener = logging.handlers.QueueListener(mqtt_queue, mqtt_file_handler)
mqtt_listener.start()

# Steuve control command logger
steuve_control_command_queue = queue.Queue()
steuve_control_command_queue_handler = logging.handlers.QueueHandler(steuve_control_command_queue)
steuve_control_command_log = logging.getLogger("steuve_control_command")
steuve_control_command_log.propagate = False
steuve_control_command_file_handler = RotatingFileHandler(
PERSISTENT_LOG_PATH + 'steuve_control_command.log', maxBytes=mb_to_bytes(80), backupCount=1)
steuve_control_command_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
steuve_control_command_log.addHandler(steuve_control_command_file_handler)

steuve_control_command_log.addHandler(steuve_control_command_queue_handler)
steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue,
steuve_control_command_file_handler)
steuve_control_command_listener.start()

# Smarthome logger
smarthome_queue = queue.Queue()
smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue)
smarthome_log_handler = RotatingFileHandler(RAMDISK_PATH + 'smarthome.log', maxBytes=mb_to_bytes(1), backupCount=1)
smarthome_log_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT))
smarthome_log_handler.addFilter(functools.partial(filter_pos, "smarthome"))
smarthome_log_handler.addFilter(RedactingFilter())
logging.getLogger().addHandler(smarthome_log_handler)
logging.getLogger().addHandler(smarthome_queue_handler)
smarthome_listener = logging.handlers.QueueListener(smarthome_queue, smarthome_log_handler)
smarthome_listener.start()

# SoC logger
soc_queue = queue.Queue()
soc_queue_handler = logging.handlers.QueueHandler(soc_queue)
soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1)
soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
soc_log_handler.addFilter(functools.partial(filter_pos, "soc"))
soc_log_handler.addFilter(RedactingFilter())
logging.getLogger().addHandler(soc_log_handler)
logging.getLogger().addHandler(soc_queue_handler)
soc_listener = logging.handlers.QueueListener(soc_queue, soc_log_handler)
soc_listener.start()

# Internal chargepoint logger
internal_chargepoint_queue = queue.Queue()
internal_chargepoint_queue_handler = logging.handlers.QueueHandler(internal_chargepoint_queue)
internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log',
maxBytes=mb_to_bytes(1),
backupCount=1)
internal_chargepoint_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint"))
internal_chargepoint_log_handler.addFilter(RedactingFilter())
logging.getLogger().addHandler(internal_chargepoint_log_handler)

logging.getLogger().addHandler(internal_chargepoint_queue_handler)
internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue,
internal_chargepoint_log_handler)
internal_chargepoint_listener.start()

# urllib3 logger
urllib3_queue = queue.Queue()
urllib3_queue_handler = logging.handlers.QueueHandler(urllib3_queue)
urllib3_log = logging.getLogger("urllib3.connectionpool")
urllib3_log.propagate = True
urllib3_file_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1)
urllib3_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED))
urllib3_file_handler.addFilter(RedactingFilter())
urllib3_file_handler.addFilter(functools.partial(filter_pos, "soc"))
urllib3_log.addHandler(urllib3_file_handler)
urllib3_log.addHandler(urllib3_queue_handler)
urllib3_listener = logging.handlers.QueueListener(urllib3_queue, urllib3_file_handler)
urllib3_listener.start()

logging.getLogger("pymodbus").setLevel(logging.WARNING)
logging.getLogger("uModbus").setLevel(logging.WARNING)
Expand Down