From c95a05cd4d9f9e1ef934fa3b4416cded9ee4efcf Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 24 Sep 2025 07:44:51 +0200 Subject: [PATCH 01/16] draft --- packages/control/process.py | 1 + packages/helpermodules/logger.py | 13 ++++ .../helpermodules/utils/_thread_handler.py | 2 + packages/main.py | 62 +++++++++++++++++++ packages/modules/loadvars.py | 5 +- packages/modules/update_soc.py | 2 + 6 files changed, 84 insertions(+), 1 deletion(-) diff --git a/packages/control/process.py b/packages/control/process.py index f4d06f68b4..96dee1d725 100644 --- a/packages/control/process.py +++ b/packages/control/process.py @@ -106,6 +106,7 @@ def process_algorithm_results(self) -> None: name=f"set output io{io.config.id}")) if modules_threads: joined_thread_handler(modules_threads, 3) + modules_threads.clear() except Exception: log.exception("Fehler im Process-Modul") diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index 2f5258dba4..21dde5d9a1 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -275,6 +275,19 @@ def mb_to_bytes(megabytes: int) -> int: steuve_control_command_file_handler) steuve_control_command_listener.start() + # Garbage collector logger + garbage_collector_queue = queue.Queue() + garbage_collector_queue_handler = logging.handlers.QueueHandler(garbage_collector_queue) + garbage_collector_log = logging.getLogger("garbage_collector") + garbage_collector_log.propagate = False + garbage_collector_file_handler = RotatingFileHandler( + RAMDISK_PATH + 'garbage_collector.log', maxBytes=mb_to_bytes(0.5), backupCount=1) + garbage_collector_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) + garbage_collector_log.addHandler(garbage_collector_queue_handler) + garbage_collector_listener = logging.handlers.QueueListener(garbage_collector_queue, + garbage_collector_file_handler) + garbage_collector_listener.start() + # Smarthome logger smarthome_queue = queue.Queue() smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue) diff --git a/packages/helpermodules/utils/_thread_handler.py b/packages/helpermodules/utils/_thread_handler.py index cea839dd20..d8744dbbc8 100644 --- a/packages/helpermodules/utils/_thread_handler.py +++ b/packages/helpermodules/utils/_thread_handler.py @@ -42,6 +42,8 @@ def split_chunks(to_split, n): if thread.is_alive(): log.error(f"{thread.name} konnte nicht innerhalb des Timeouts abgearbeitet werden.") not_finished_threads.append(thread.name) + # Entferne alle beendeten Threads aus der übergebenen Liste + threads[:] = [t for t in threads if t.is_alive()] return not_finished_threads diff --git a/packages/main.py b/packages/main.py index 2fbd15b490..e08919bd57 100755 --- a/packages/main.py +++ b/packages/main.py @@ -8,11 +8,16 @@ import threading import sys import functools +import gc # GC-Modul importieren +import collections # als erstes logging initialisieren, damit auch ImportError geloggt werden logger.setup_logging() log = logging.getLogger() +# GC-Logger einrichten +gc_logger = logging.getLogger("garbage_collector") + from pathlib import Path from random import randrange import schedule @@ -167,6 +172,9 @@ def handler_with_control_interval(): try: handler_with_control_interval() logger.write_logs_to_file("main") + write_gc_stats() + #print_active_threads_and_referrers() + analyze_function_origins() finally: self.__release_lock("handler10Sec") except Exception: @@ -255,6 +263,58 @@ def handler_hour(self): except Exception: log.exception("Fehler im Main-Modul") +def write_gc_stats(): + """Schreibt GC-Statistiken in eine Datei über den eigenen Logger.""" + try: + gc.collect() + objs = gc.get_objects() + type_counter = collections.Counter(type(obj).__name__ for obj in objs) + most_common = type_counter.most_common(10) + stats = { + "garbage": len(gc.garbage), + "objects": len(objs), + "counts": gc.get_count(), + "top_types": most_common, + } + gc_logger.info(stats) + except Exception: + log.exception("Fehler beim Schreiben der GC-Statistiken") + +def analyze_function_origins(): + """Analysiert, woher die Funktionsobjekte im Speicher stammen.""" + import types + gc.collect() + objs = gc.get_objects() + origins = {} + for obj in objs: + if isinstance(obj, types.FunctionType): + try: + key = ( + getattr(obj, "__module__", None), + getattr(obj, "__qualname__", None), + getattr(obj, "__code__", None) and obj.__code__.co_filename, + getattr(obj, "__code__", None) and obj.__code__.co_firstlineno, + ) + origins[key] = origins.get(key, 0) + 1 + except Exception: + pass + # Die 10 häufigsten Ursprünge loggen + top = sorted(origins.items(), key=lambda x: x[1], reverse=True)[:10] + for (mod, name, file, line), count in top: + gc_logger.info(f"Function origin: {mod}.{name} in {file}:{line} -> {count}x") + +def print_active_threads_and_referrers(max_referrers=3): + """Zeigt alle aktiven Thread-Objekte und eine begrenzte Anzahl Referrers an.""" + for obj in gc.get_objects(): + if isinstance(obj, threading.Thread) and not obj.is_alive(): + gc_logger.info(f"Thread: {obj} (Name: {obj.name}, Alive: {obj.is_alive()}, id={id(obj)})") + referrers = gc.get_referrers(obj) + gc_logger.info(f" Referrers count: {len(referrers)}") + for ref in referrers[:max_referrers]: + gc_logger.info(f" Referrer type: {type(ref)}, id={id(ref)}, repr={repr(ref)[:400]}") + +# Beispiel: Manuell aufrufen, wenn du eine Analyse willst +# analyze_function_origins() def schedule_jobs(): [schedule.every().minute.at(f":{i:02d}").do(smarthome_handler).tag("algorithm") for i in range(0, 60, 5)] @@ -269,6 +329,8 @@ def schedule_jobs(): [schedule.every().minute.at(f":{i:02d}").do(handler.handler10Sec).tag("algorithm") for i in range(0, 60, 10)] # 30 Sekunden Handler, der die Locks überwacht, Deadlocks erkennt, loggt und ggf. den Prozess beendet schedule.every(30).seconds.do(handler.monitor_handler_locks, max_runtime=600) + # GC-Analyse alle 5 Minuten + schedule.every(5).minutes.do(write_gc_stats) try: diff --git a/packages/modules/loadvars.py b/packages/modules/loadvars.py index ef1b07618a..dd1f91220e 100644 --- a/packages/modules/loadvars.py +++ b/packages/modules/loadvars.py @@ -51,7 +51,9 @@ def _set_values(self) -> List[str]: args=(), name=f"set values cp{cp.chargepoint_module.config.id}")) except Exception: log.exception(f"Fehler im loadvars-Modul bei Element {cp.num}") - return joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) + result = joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) + modules_threads.clear() + return result def _update_values_of_level(self, elements, not_finished_threads: List[str]) -> None: """Threads, um von der niedrigsten Ebene der Hierarchie Werte ggf. miteinander zu verrechnen und zu @@ -75,6 +77,7 @@ def _update_values_of_level(self, elements, not_finished_threads: List[str]) -> except Exception: log.exception(f"Fehler im loadvars-Modul bei Element {element}") joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) + modules_threads.clear() def thread_without_set_value(self, modules_threads: List[Thread], diff --git a/packages/modules/update_soc.py b/packages/modules/update_soc.py index 998f4caab7..7b5ccdc93a 100644 --- a/packages/modules/update_soc.py +++ b/packages/modules/update_soc.py @@ -34,9 +34,11 @@ def update(self) -> None: clear_in_memory_log_handler("soc") threads_update, threads_store = self._get_threads() joined_thread_handler(threads_update, 300) + threads_update.clear() wait_for_module_update_completed(self.event_vehicle_update_completed, topic) # threads_store = self._filter_failed_store_threads(threads_store) joined_thread_handler(threads_store, data.data.general_data.data.control_interval/3) + threads_store.clear() wait_for_module_update_completed(self.event_vehicle_update_completed, topic) write_logs_to_file("soc") except Exception: From f607d2e25f00028608768983d02913c72b0e4889 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Fri, 26 Sep 2025 10:17:41 +0200 Subject: [PATCH 02/16] add trace_malloc --- packages/helpermodules/logger.py | 15 ++++- packages/main.py | 104 ++++++++++++++++++++----------- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index 21dde5d9a1..98069826cc 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -212,7 +212,7 @@ def mb_to_bytes(megabytes: int) -> int: # Main logger log_queue = queue.Queue() queue_handler = logging.handlers.QueueHandler(log_queue) - main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5.5), backupCount=4) + main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5), backupCount=4) main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) main_file_handler.addFilter(RedactingFilter()) in_memory_log_handlers["main"] = InMemoryLogHandler(main_file_handler) @@ -288,6 +288,19 @@ def mb_to_bytes(megabytes: int) -> int: garbage_collector_file_handler) garbage_collector_listener.start() + # tracemalloc logger + tracemalloc_queue = queue.Queue() + tracemalloc_queue_handler = logging.handlers.QueueHandler(tracemalloc_queue) + tracemalloc_log = logging.getLogger("tracemalloc") + tracemalloc_log.propagate = False + tracemalloc_file_handler = RotatingFileHandler( + RAMDISK_PATH + 'tracemalloc.log', maxBytes=mb_to_bytes(0.5), backupCount=1) + tracemalloc_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) + tracemalloc_log.addHandler(tracemalloc_queue_handler) + tracemalloc_listener = logging.handlers.QueueListener(tracemalloc_queue, + tracemalloc_file_handler) + tracemalloc_listener.start() + # Smarthome logger smarthome_queue = queue.Queue() smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue) diff --git a/packages/main.py b/packages/main.py index e08919bd57..fc7a2bcbb6 100755 --- a/packages/main.py +++ b/packages/main.py @@ -10,13 +10,26 @@ import functools import gc # GC-Modul importieren import collections - +import tracemalloc # als erstes logging initialisieren, damit auch ImportError geloggt werden logger.setup_logging() log = logging.getLogger() # GC-Logger einrichten gc_logger = logging.getLogger("garbage_collector") +tracemalloc_logger = logging.getLogger("tracemalloc") +def gc_callback(phase, info): + objs = gc.get_objects() + type_counter = collections.Counter(type(obj).__name__ for obj in objs) + most_common = type_counter.most_common(10) + stats = { + "garbage": len(gc.garbage), + "objects": len(objs), + "counts": gc.get_count(), + "top_types": most_common, + } + gc_logger.info(f"GC-Phase: {phase}, Info: {info}, Stats: {stats}") + from pathlib import Path from random import randrange @@ -174,7 +187,8 @@ def handler_with_control_interval(): logger.write_logs_to_file("main") write_gc_stats() #print_active_threads_and_referrers() - analyze_function_origins() + #analyze_function_origins() + log_memory_usage() finally: self.__release_lock("handler10Sec") except Exception: @@ -226,6 +240,7 @@ def handler5Min(self): general_internal_chargepoint_handler.internal_chargepoint_handler.heartbeat = False with ChangedValuesContext(loadvars_.event_module_update_completed): sub.system_data["system"].update_ip_address() + log_memory_usage(always_log=True) except Exception: log.exception("Fehler im Main-Modul") @@ -263,10 +278,27 @@ def handler_hour(self): except Exception: log.exception("Fehler im Main-Modul") + +old_memory_usage: int + +def log_memory_usage(always_log=False): + global old_memory_usage + with open("/proc/self/status") as f: + for line in f: + if line.startswith("VmRSS:"): + mem_kb = int(line.split()[1]) + memory_usage = mem_kb / 1024 # MB + if old_memory_usage + 30 < memory_usage or always_log: + tracemalloc_logger.debug(f"Speicherverbrauch: {memory_usage:.2f} MB, vorheriger: {old_memory_usage:.2f} MB") + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('lineno') + for stat in top_stats[:10]: + tracemalloc_logger.debug(stat) + old_memory_usage = memory_usage + def write_gc_stats(): """Schreibt GC-Statistiken in eine Datei über den eigenen Logger.""" try: - gc.collect() objs = gc.get_objects() type_counter = collections.Counter(type(obj).__name__ for obj in objs) most_common = type_counter.most_common(10) @@ -280,38 +312,38 @@ def write_gc_stats(): except Exception: log.exception("Fehler beim Schreiben der GC-Statistiken") -def analyze_function_origins(): - """Analysiert, woher die Funktionsobjekte im Speicher stammen.""" - import types - gc.collect() - objs = gc.get_objects() - origins = {} - for obj in objs: - if isinstance(obj, types.FunctionType): - try: - key = ( - getattr(obj, "__module__", None), - getattr(obj, "__qualname__", None), - getattr(obj, "__code__", None) and obj.__code__.co_filename, - getattr(obj, "__code__", None) and obj.__code__.co_firstlineno, - ) - origins[key] = origins.get(key, 0) + 1 - except Exception: - pass - # Die 10 häufigsten Ursprünge loggen - top = sorted(origins.items(), key=lambda x: x[1], reverse=True)[:10] - for (mod, name, file, line), count in top: - gc_logger.info(f"Function origin: {mod}.{name} in {file}:{line} -> {count}x") - -def print_active_threads_and_referrers(max_referrers=3): - """Zeigt alle aktiven Thread-Objekte und eine begrenzte Anzahl Referrers an.""" - for obj in gc.get_objects(): - if isinstance(obj, threading.Thread) and not obj.is_alive(): - gc_logger.info(f"Thread: {obj} (Name: {obj.name}, Alive: {obj.is_alive()}, id={id(obj)})") - referrers = gc.get_referrers(obj) - gc_logger.info(f" Referrers count: {len(referrers)}") - for ref in referrers[:max_referrers]: - gc_logger.info(f" Referrer type: {type(ref)}, id={id(ref)}, repr={repr(ref)[:400]}") +# def analyze_function_origins(): +# """Analysiert, woher die Funktionsobjekte im Speicher stammen.""" +# import types +# gc.collect() +# objs = gc.get_objects() +# origins = {} +# for obj in objs: +# if isinstance(obj, types.FunctionType): +# try: +# key = ( +# getattr(obj, "__module__", None), +# getattr(obj, "__qualname__", None), +# getattr(obj, "__code__", None) and obj.__code__.co_filename, +# getattr(obj, "__code__", None) and obj.__code__.co_firstlineno, +# ) +# origins[key] = origins.get(key, 0) + 1 +# except Exception: +# pass +# # Die 10 häufigsten Ursprünge loggen +# top = sorted(origins.items(), key=lambda x: x[1], reverse=True)[:10] +# for (mod, name, file, line), count in top: +# gc_logger.info(f"Function origin: {mod}.{name} in {file}:{line} -> {count}x") + +# def print_active_threads_and_referrers(max_referrers=3): +# """Zeigt alle aktiven Thread-Objekte und eine begrenzte Anzahl Referrers an.""" +# for obj in gc.get_objects(): +# if isinstance(obj, threading.Thread) and not obj.is_alive(): +# gc_logger.info(f"Thread: {obj} (Name: {obj.name}, Alive: {obj.is_alive()}, id={id(obj)})") +# referrers = gc.get_referrers(obj) +# gc_logger.info(f" Referrers count: {len(referrers)}") +# for ref in referrers[:max_referrers]: +# gc_logger.info(f" Referrer type: {type(ref)}, id={id(ref)}, repr={repr(ref)[:400]}") # Beispiel: Manuell aufrufen, wenn du eine Analyse willst # analyze_function_origins() @@ -335,6 +367,8 @@ def schedule_jobs(): try: log.debug("Start openWB2.service") + tracemalloc.start() + old_memory_usage = 0 loadvars_ = loadvars.Loadvars() data.data_init(loadvars_.event_module_update_completed) update_config.UpdateConfig().update() From 390df07bba0c0ba8f90ede69d3ae6e5751d8c68d Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Fri, 26 Sep 2025 10:26:06 +0200 Subject: [PATCH 03/16] without list clean up (as in master) --- packages/control/process.py | 1 - packages/helpermodules/utils/_thread_handler.py | 2 -- packages/modules/loadvars.py | 2 -- packages/modules/update_soc.py | 2 -- 4 files changed, 7 deletions(-) diff --git a/packages/control/process.py b/packages/control/process.py index 96dee1d725..f4d06f68b4 100644 --- a/packages/control/process.py +++ b/packages/control/process.py @@ -106,7 +106,6 @@ def process_algorithm_results(self) -> None: name=f"set output io{io.config.id}")) if modules_threads: joined_thread_handler(modules_threads, 3) - modules_threads.clear() except Exception: log.exception("Fehler im Process-Modul") diff --git a/packages/helpermodules/utils/_thread_handler.py b/packages/helpermodules/utils/_thread_handler.py index d8744dbbc8..cea839dd20 100644 --- a/packages/helpermodules/utils/_thread_handler.py +++ b/packages/helpermodules/utils/_thread_handler.py @@ -42,8 +42,6 @@ def split_chunks(to_split, n): if thread.is_alive(): log.error(f"{thread.name} konnte nicht innerhalb des Timeouts abgearbeitet werden.") not_finished_threads.append(thread.name) - # Entferne alle beendeten Threads aus der übergebenen Liste - threads[:] = [t for t in threads if t.is_alive()] return not_finished_threads diff --git a/packages/modules/loadvars.py b/packages/modules/loadvars.py index dd1f91220e..e46d4e2150 100644 --- a/packages/modules/loadvars.py +++ b/packages/modules/loadvars.py @@ -52,7 +52,6 @@ def _set_values(self) -> List[str]: except Exception: log.exception(f"Fehler im loadvars-Modul bei Element {cp.num}") result = joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) - modules_threads.clear() return result def _update_values_of_level(self, elements, not_finished_threads: List[str]) -> None: @@ -77,7 +76,6 @@ def _update_values_of_level(self, elements, not_finished_threads: List[str]) -> except Exception: log.exception(f"Fehler im loadvars-Modul bei Element {element}") joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) - modules_threads.clear() def thread_without_set_value(self, modules_threads: List[Thread], diff --git a/packages/modules/update_soc.py b/packages/modules/update_soc.py index 7b5ccdc93a..998f4caab7 100644 --- a/packages/modules/update_soc.py +++ b/packages/modules/update_soc.py @@ -34,11 +34,9 @@ def update(self) -> None: clear_in_memory_log_handler("soc") threads_update, threads_store = self._get_threads() joined_thread_handler(threads_update, 300) - threads_update.clear() wait_for_module_update_completed(self.event_vehicle_update_completed, topic) # threads_store = self._filter_failed_store_threads(threads_store) joined_thread_handler(threads_store, data.data.general_data.data.control_interval/3) - threads_store.clear() wait_for_module_update_completed(self.event_vehicle_update_completed, topic) write_logs_to_file("soc") except Exception: From 2d4f58f3739ba78565f7e448264c04818dfe8cf8 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Fri, 26 Sep 2025 10:58:31 +0200 Subject: [PATCH 04/16] loglevel --- packages/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/main.py b/packages/main.py index fc7a2bcbb6..12fddb3891 100755 --- a/packages/main.py +++ b/packages/main.py @@ -28,7 +28,7 @@ def gc_callback(phase, info): "counts": gc.get_count(), "top_types": most_common, } - gc_logger.info(f"GC-Phase: {phase}, Info: {info}, Stats: {stats}") + gc_logger.error(f"GC-Phase: {phase}, Info: {info}, Stats: {stats}") from pathlib import Path @@ -289,11 +289,11 @@ def log_memory_usage(always_log=False): mem_kb = int(line.split()[1]) memory_usage = mem_kb / 1024 # MB if old_memory_usage + 30 < memory_usage or always_log: - tracemalloc_logger.debug(f"Speicherverbrauch: {memory_usage:.2f} MB, vorheriger: {old_memory_usage:.2f} MB") + tracemalloc_logger.error(f"Speicherverbrauch: {memory_usage:.2f} MB, vorheriger: {old_memory_usage:.2f} MB") snapshot = tracemalloc.take_snapshot() top_stats = snapshot.statistics('lineno') for stat in top_stats[:10]: - tracemalloc_logger.debug(stat) + tracemalloc_logger.error(stat) old_memory_usage = memory_usage def write_gc_stats(): @@ -308,7 +308,7 @@ def write_gc_stats(): "counts": gc.get_count(), "top_types": most_common, } - gc_logger.info(stats) + gc_logger.error(stats) except Exception: log.exception("Fehler beim Schreiben der GC-Statistiken") From d77ef5141b36d24a5cf0d1246e004c3ecc90d348 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Mon, 13 Oct 2025 15:26:48 +0200 Subject: [PATCH 05/16] use filehandler for smarthome --- .../smarthome/avmhomeautomation/avmcommon.py | 38 +++---------- packages/modules/smarthome/fronius/watt.py | 9 +-- packages/modules/smarthome/json/watt.py | 9 +-- packages/modules/smarthome/mqtt/off.py | 5 +- packages/modules/smarthome/mqtt/on.py | 5 +- packages/modules/smarthome/mqtt/watt.py | 5 +- packages/modules/smarthome/mystrom/watt.py | 8 ++- packages/modules/smarthome/nibe/watt.py | 8 ++- packages/modules/smarthome/smaem/watt.py | 55 ++++++++----------- packages/modules/smarthome/tasmota/watt.py | 5 +- packages/modules/smarthome/viessmann/off.py | 26 +++------ packages/modules/smarthome/viessmann/on.py | 27 +++------ packages/modules/smarthome/viessmann/watt.py | 22 ++++---- packages/modules/smarthome/we514/watt.py | 7 +-- 14 files changed, 88 insertions(+), 141 deletions(-) diff --git a/packages/modules/smarthome/avmhomeautomation/avmcommon.py b/packages/modules/smarthome/avmhomeautomation/avmcommon.py index 8124f3d2c1..4dd27c5c94 100644 --- a/packages/modules/smarthome/avmhomeautomation/avmcommon.py +++ b/packages/modules/smarthome/avmhomeautomation/avmcommon.py @@ -42,9 +42,8 @@ def __init__(self): if os.path.isfile(CACHEFILE): self.logMessage(LOGLEVELDEBUG, "found an AVM cache file, trying to load") try: - f = open(CACHEFILE, 'r') - self.cache = json.loads(f.read().strip()) - f.close() + with open(CACHEFILE, 'r') as f: + self.cache = json.loads(f.read().strip()) except Exception as e: self.logMessage(LOGLEVELDEBUG, "unable to load cache file: %s" % (e)) @@ -65,9 +64,8 @@ def writeCacheToRamdisk(self): else: cacheToWrite[login] = self.cache[login] try: - f = open(CACHEFILE, 'w') - json.dump(cacheToWrite, f) - f.close() + with open(CACHEFILE, 'w') as f: + json.dump(cacheToWrite, f) except Exception as e: self.logMessage(LOGLEVELDEBUG, "unable to write cache file: %s" % (e)) @@ -166,28 +164,7 @@ def connect(self): # logMessage writes a message to the logfile for the smarthome device. def logMessage(self, level, message): - if level < self.loglevel: - return - now = time.localtime() # getstruct_time - time_string = time.strftime("%Y-%m-%d %H:%M:%S", now) - logfile_string = '/var/www/html/openWB/ramdisk/smarthome.log' - try: - if os.path.isfile(logfile_string): - f = open(logfile_string, 'a', encoding='utf8') - else: - f = open(logfile_string, 'w', encoding='utf8') - prefix = "" - if level == LOGLEVELDEBUG: - prefix = "[DEBUG] " - if level == LOGLEVELINFO: - prefix = "[INFO] " - if level == LOGLEVELERROR: - prefix = "[ERROR] " - log.debug('%s: (%s) AVM (actor: %s) %s%s' % - (time_string, self.devicenumber, self.switchname, prefix, message), file=f) - f.close() - except IOError: - pass + log.debug(f'({self.devicenumber}) AVM (actor: {self.switchname}) {message}') # getDevicesDict returns a dictionary that maps defined actor names to its # unique hardware ID (called "AIN": "Actuator Identification Number") and @@ -362,9 +339,8 @@ def getActualPower(self): outFileString = '/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(self.devicenumber) self.logMessage(LOGLEVELDEBUG, "handing answer back to smarthomehandler via %s" % (outFileString)) try: - f1 = open(outFileString, 'w') - json.dump(answer, f1) - f1.close() + with open(outFileString, 'w') as f1: + json.dump(answer, f1) except IOError as e: self.logMessage(LOGLEVELERROR, "error writing power result %s" % (e)) log.debug(answer) # dump answer to stdout if file cannot be written diff --git a/packages/modules/smarthome/fronius/watt.py b/packages/modules/smarthome/fronius/watt.py index ca0eb4e505..4470418ec8 100644 --- a/packages/modules/smarthome/fronius/watt.py +++ b/packages/modules/smarthome/fronius/watt.py @@ -1,9 +1,12 @@ #!/usr/bin/python3 +import logging import sys import json import jq import urllib.request +log = logging.getLogger(__name__) + devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) # IP-ADresse des Fronius Wechselrichters, mit dem der Zähler kommuniziert smid = int(sys.argv[3]) # ID des Zählers im Wechselrichter (Hauptzähler 0, weitere fortlaufend) @@ -27,7 +30,5 @@ except Exception: powerc = 0 -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -answer = '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}' -json.dump(answer, f1) -f1.close() +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/json/watt.py b/packages/modules/smarthome/json/watt.py index efe2381ad9..5e82995631 100644 --- a/packages/modules/smarthome/json/watt.py +++ b/packages/modules/smarthome/json/watt.py @@ -1,9 +1,12 @@ #!/usr/bin/python3 +import logging import sys import json import jq import urllib.request +log = logging.getLogger(__name__) + devicenumber = str(sys.argv[1]) # Abfrage-URL, die die .json Antwort liefert. Z.B. # "http://192.168.0.150/solar_api/v1/GetMeterRealtimeData.cgi?Scope=Device&DeviceID=1" @@ -26,7 +29,5 @@ except Exception: powerc = 0 -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -answer = '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}' -json.dump(answer, f1) -f1.close() +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/mqtt/off.py b/packages/modules/smarthome/mqtt/off.py index d7cea04010..672f1e9c28 100644 --- a/packages/modules/smarthome/mqtt/off.py +++ b/packages/modules/smarthome/mqtt/off.py @@ -35,6 +35,5 @@ def on_message(client, userdata, msg) -> None: client.disconnect() file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' pvmodus = 0 -f = open(file_stringpv, 'w') -f.write(str(pvmodus)) -f.close() +with open(file_stringpv, 'w') as f: + f.write(str(pvmodus)) diff --git a/packages/modules/smarthome/mqtt/on.py b/packages/modules/smarthome/mqtt/on.py index b5f70ed825..781d850a81 100644 --- a/packages/modules/smarthome/mqtt/on.py +++ b/packages/modules/smarthome/mqtt/on.py @@ -34,6 +34,5 @@ def on_message(client, userdata, msg) -> None: client.loop(timeout=2.0) client.disconnect() file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' -f = open(file_stringpv, 'w') -f.write(str(1)) -f.close() +with open(file_stringpv, 'w') as f: + f.write(str(1)) diff --git a/packages/modules/smarthome/mqtt/watt.py b/packages/modules/smarthome/mqtt/watt.py index d590c36417..d1cb945fbc 100644 --- a/packages/modules/smarthome/mqtt/watt.py +++ b/packages/modules/smarthome/mqtt/watt.py @@ -69,9 +69,8 @@ def on_message(client, userdata, msg) -> None: # PV-Modus pvmodus = 0 if os.path.isfile(file_stringpv): - f = open(file_stringpv, 'r') - pvmodus = int(f.read()) - f.close() + with open(file_stringpv, 'r') as f: + pvmodus = int(f.read()) answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) answer += ',"on":' + str(pvmodus) + ',"temp0":' + str(tempa) answer += ',"temp1":' + str(tempb) + ',"temp2":' + str(tempc) + '}' diff --git a/packages/modules/smarthome/mystrom/watt.py b/packages/modules/smarthome/mystrom/watt.py index 3ae83cf393..1a4faf97bf 100644 --- a/packages/modules/smarthome/mystrom/watt.py +++ b/packages/modules/smarthome/mystrom/watt.py @@ -1,9 +1,12 @@ #!/usr/bin/python3 +import logging import sys import time import json import urllib.request +log = logging.getLogger(__name__) + named_tuple = time.localtime() # getstruct_time time_string = time.strftime("%m/%d/%Y, %H:%M:%S mystrom watty.py", named_tuple) devicenumber = str(sys.argv[1]) @@ -21,6 +24,5 @@ powerc = 0 answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + \ str(relais) + ',"temp0":' + str(temp) + '} ' -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -json.dump(answer, f1) -f1.close() +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + log.debug(answer) diff --git a/packages/modules/smarthome/nibe/watt.py b/packages/modules/smarthome/nibe/watt.py index 3b9bc3f286..99ecca59a1 100644 --- a/packages/modules/smarthome/nibe/watt.py +++ b/packages/modules/smarthome/nibe/watt.py @@ -1,8 +1,11 @@ #!/usr/bin/python3 +import logging import sys import json from pymodbus.client.sync import ModbusTcpClient +log = logging.getLogger(__name__) + # get variables from arguments devicenumber = str(sys.argv[1]) # SmartHome device number SERVER_HOST = str(sys.argv[2]) # IP of server to connect to @@ -29,8 +32,7 @@ CurrentPower = None # Handle error case answer = '{"power":' + str(CurrentPower) + '}' -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -json.dump(answer, f1) -f1.close() +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + json.dump(answer, f1) client.close() # clean disconnect from modbus server diff --git a/packages/modules/smarthome/smaem/watt.py b/packages/modules/smarthome/smaem/watt.py index bfb767a89a..f31ef9c3d2 100644 --- a/packages/modules/smarthome/smaem/watt.py +++ b/packages/modules/smarthome/smaem/watt.py @@ -33,7 +33,6 @@ import os import datetime import time -import json import signal import socket import struct @@ -68,7 +67,7 @@ def abortprogram(signal, frame): timefile = '/var/www/html/openWB/ramdisk/smarthome_device_ret' + \ str(devicenumber) + '_time' # Dummy file needed for timestamp of last metering # Logfile for additional output beside of the smarthome.log -debugfile = open('/var/www/html/openWB/ramdisk/smaem.log', 'a', newline='\r\n') + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -108,9 +107,9 @@ def abortprogram(signal, frame): "HomeManager (2) is sending in the network.") emparts = decode_speedwire(sock_data) -debugfile.write(str(datetime.datetime.now()) + ': smaserial: #' + str(smaserial) + '# - Current SMA serial number:#' + - str(emparts['serial']) + '# - watt:#' + str(int(emparts.get("pconsume"))) + '# - wattc:#' + - str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) + '#\n') +log.debug('SMA:: smaserial: #' + str(smaserial) + '# - Current SMA serial number:#' + + str(emparts['serial']) + '# - watt:#' + str(int(emparts.get("pconsume"))) + '# - wattc:#' + + str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) + '#') # Remember: We assume that beside of our EnergyMeter there are more SMA devices present (like HomeManager 2.0 or other # EnergyMeter) - so must not accept any data or smaserial = None @@ -118,8 +117,8 @@ def abortprogram(signal, frame): # our output variables watt = str(int(emparts.get("pconsume"))) wattc = str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) - debugfile.write(str(datetime.datetime.now()) + ': 1 - Our SMA EM ' + str(smaserial) + - ' is sending, everything fine. watt: #' + str(watt) + '# - wattc: #' + str(wattc) + '#\n') + log.debug('SMA:: 1 - Our SMA EM ' + str(smaserial) + + ' is sending, everything fine. watt: #' + str(watt) + '# - wattc: #' + str(wattc) + '#') # Scenario 2: Our EnergyMeter is not sending but we have a returnfile which is older than n seconds (parameter # secondssincelastmetering) elif ((os.path.isfile(returnfile)) and @@ -128,29 +127,27 @@ def abortprogram(signal, frame): # We set "0" as current Power Consume (pconsume) and (from the existing ret-file) the last value for the Power # Consume Counter (pconsumecounter) watt = '0' - ret = open(returnfile, 'r') - lastvalues = ret.read() - ret.close() + with open(returnfile, 'r') as ret: + lastvalues = ret.read() timesincelastmetering = int(round(time.time(), 0)-lastmodificationtime) wattc = lastvalues[int(lastvalues.rfind('powerc')) + 9:lastvalues.find('}')] - debugfile.write(str(datetime.datetime.now()) + ': 2 - Debug: time.time(): #' + str(round(time.time(), 0)) + - '# - lastmodificationtime: #' + str(lastmodificationtime) + - '# - timesincelastmetering: #' + str(timesincelastmetering) + - '# - int(secondssincelastmetering): #' + str(int(secondssincelastmetering)) + '#\n') - debugfile.write(str(datetime.datetime.now()) + ': 2 - We create a fake ret-file. watt: #' + str(watt) + - '# - wattc: #' + str( + log.debug('SMA:: 2 - Debug: time.time(): #' + str(round(time.time(), 0)) + + '# - lastmodificationtime: #' + str(lastmodificationtime) + + '# - timesincelastmetering: #' + str(timesincelastmetering) + + '# - int(secondssincelastmetering): #' + str(int(secondssincelastmetering)) + '#') + log.debug('SMA:: 2 - We create a fake ret-file. watt: #' + str(watt) + + '# - wattc: #' + str( wattc) + '# - lastvalues: #' + lastvalues + '# - int-lastvalues.rfind-powerc: #' + - str((lastvalues.rfind('powerc'))) + '#\n') + str((lastvalues.rfind('powerc'))) + '#') # Scenario 3: Our EnergyMeter is not sending but we have a returnfile which is younger than n seconds # (parameter secondssincelastmetering) elif ((os.path.isfile(returnfile)) and (int((round(time.time(), 0)-lastmodificationtime)) < int(secondssincelastmetering))): # We have a ret-file which is younger than n seconds. We do nothing as the existing ret-file is good enough. - debugfile.write(str(datetime.datetime.now()) + - ': 3 - The existing ret-file is fine enough. round(time.time(),0): #' + str(round(time.time(), 0)) - + '# - lastmodificationtime: #' + str(lastmodificationtime) + '# - secondssincelastmetering: #' - + str(secondssincelastmetering) + '#\n') + log.debug('SMA:: 3 - The existing ret-file is fine enough. round(time.time(),0): #' + str(round(time.time(), 0)) + + '# - lastmodificationtime: #' + str(lastmodificationtime) + '# - secondssincelastmetering: #' + + str(secondssincelastmetering) + '#') sys.exit("Module SMAEM: No data received but we have historical data which is younger than " + str(secondssincelastmetering) + " seconds.") else: @@ -162,20 +159,16 @@ def abortprogram(signal, frame): # Module SMAEM: No data received and no historical data since boot time # Leistungsmessung smaem [...] Fehlermeldung: [Errno 2] No such file or directory: # '/var/www/html/openWB/ramdisk/smarthome_device_ret1' - debugfile.write(str(datetime.datetime.now()) + ': 4 - No data received and no historical data since boottime\n') + log.debug('SMA:: 4 - No data received and no historical data since boottime') sys.exit(str(datetime.datetime.now()) + ": Module SMAEM: No data received and no historical data since boottime") # General output section answer = '{"power":' + watt + ',"powerc":' + wattc + '}' -f = open(returnfile, 'w') -json.dump(answer, f) -f.close() -t = open(timefile, 'w') -t.write(str(datetime.datetime.now()) + - ': File is created by Smarthome module SMAEM for validating the timestamp of the last return-file creation.') -t.close() +with open(timefile, 'w') as t: + t.write(str(datetime.datetime.now()) + + ': File is created by Smarthome module SMAEM for validating ' + 'the timestamp of the last return-file creation.') -debugfile.write(str(datetime.datetime.now()) + ': 99 - Output answer: #' + answer + '#\n') -debugfile.close() +log.debug('SMA:: 99 - Output answer: #' + answer + '#') diff --git a/packages/modules/smarthome/tasmota/watt.py b/packages/modules/smarthome/tasmota/watt.py index b5f9a54832..533d796d39 100644 --- a/packages/modules/smarthome/tasmota/watt.py +++ b/packages/modules/smarthome/tasmota/watt.py @@ -26,6 +26,5 @@ relais = 1 powerc = 0 answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + str(relais) + '} ' -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -json.dump(answer, f1) -f1.close() +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + json.dump(answer, f1) diff --git a/packages/modules/smarthome/viessmann/off.py b/packages/modules/smarthome/viessmann/off.py index 12cf698ef9..d294441d9e 100644 --- a/packages/modules/smarthome/viessmann/off.py +++ b/packages/modules/smarthome/viessmann/off.py @@ -1,14 +1,10 @@ #!/usr/bin/python3 import sys -import os -import time from pymodbus.client.sync import ModbusTcpClient import logging log = logging.getLogger(__name__) -named_tuple = time.localtime() # getstruct_time -time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann off.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) @@ -22,22 +18,14 @@ # coils read write boolean # register start 00000 # -file_string = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_viessmann.log' -file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' -if os.path.isfile(file_string): - f = open(file_string, 'a') -else: - f = open(file_string, 'w') -log.debug('%s devicenr %s ipadr %s ueberschuss %6d try to connect (modbus)' % - (time_string, devicenumber, ipadr, uberschuss), file=f) +log.debug(f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " + f"ueberschuss {uberschuss:6d} try to connect (modbus)") client = ModbusTcpClient(ipadr, port=502) rq = client.write_coil(16, False, unit=1) -log.debug(rq, file=f) +log.debug(f"[Viessmann {devicenumber}] Modbus write_coil response: {rq}") client.close() -log.debug('%s devicenr %s ipadr %s Einmalige Warmwasseraufbereitung deaktiviert CO-17 = 0 ' % - (time_string, devicenumber, ipadr), file=f) -f.close() +log.debug( + f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " + "Einmalige Warmwasseraufbereitung deaktiviert CO-17 = 0") pvmodus = 0 -f = open(file_stringpv, 'w') -f.write(str(pvmodus)) -f.close() +log.debug(f"[Viessmann {devicenumber}] PV-Modus gesetzt: {pvmodus}") diff --git a/packages/modules/smarthome/viessmann/on.py b/packages/modules/smarthome/viessmann/on.py index f85a7d6d6f..5f7415cd16 100644 --- a/packages/modules/smarthome/viessmann/on.py +++ b/packages/modules/smarthome/viessmann/on.py @@ -1,14 +1,10 @@ #!/usr/bin/python3 import sys -import os -import time from pymodbus.client.sync import ModbusTcpClient import logging log = logging.getLogger(__name__) -named_tuple = time.localtime() # getstruct_time -time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann on.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) @@ -24,21 +20,14 @@ # coils read write boolean # register start 00000 # -file_string = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_viessmann.log' -file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' -if os.path.isfile(file_string): - f = open(file_string, 'a') -else: - f = open(file_string, 'w') -log.debug('%s devicenr %s ipadr %s ueberschuss %6d try to connect (modbus)' % - (time_string, devicenumber, ipadr, uberschuss), file=f) +log.debug( + f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " + f"ueberschuss {uberschuss:6d} try to connect (modbus)") client = ModbusTcpClient(ipadr, port=502) rq = client.write_coil(16, True, unit=1) -log.debug(rq, file=f) +log.debug(f"[Viessmann {devicenumber}] Modbus write_coil response: {rq}") client.close() -log.debug('%s devicenr %s ipadr %s Einmalige Warmwasseraufbereitung aktiviert CO-17 = 1' % - (time_string, devicenumber, ipadr), file=f) -f.close() -f = open(file_stringpv, 'w') -f.write(str(1)) -f.close() +log.debug( + f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " + "Einmalige Warmwasseraufbereitung aktiviert CO-17 = 1") +log.debug(f"[Viessmann {devicenumber}] PV-Modus gesetzt: 1") diff --git a/packages/modules/smarthome/viessmann/watt.py b/packages/modules/smarthome/viessmann/watt.py index e46ec14986..5d192050aa 100644 --- a/packages/modules/smarthome/viessmann/watt.py +++ b/packages/modules/smarthome/viessmann/watt.py @@ -1,24 +1,24 @@ #!/usr/bin/python3 + import sys import os -import time import json +import logging + +log = logging.getLogger(__name__) -named_tuple = time.localtime() # getstruct_time -time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann watty.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) -file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' +file_stringpv = f'/var/www/html/openWB/ramdisk/smarthome_device_{devicenumber}_pv' # PV-Modus pvmodus = 0 if os.path.isfile(file_stringpv): - f = open(file_stringpv, 'r') - pvmodus = int(f.read()) - f.close() + with open(file_stringpv, 'r') as f: + pvmodus = int(f.read()) aktpower = 0 powerc = 0 -answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + str(pvmodus) + '} ' -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -json.dump(answer, f1) -f1.close() +answer = {"power": aktpower, "powerc": powerc, "on": pvmodus} +outfile = f'/var/www/html/openWB/ramdisk/smarthome_device_ret{devicenumber}' +with open(outfile, 'w') as f1: + json.dump(answer, f1) diff --git a/packages/modules/smarthome/we514/watt.py b/packages/modules/smarthome/we514/watt.py index 5a12363c25..f9d3afe00e 100644 --- a/packages/modules/smarthome/we514/watt.py +++ b/packages/modules/smarthome/we514/watt.py @@ -33,9 +33,8 @@ resp = client.read_holding_registers(CurrentPowerRegisterAddress, 1, unit=MODBUS_DEVICEID) CurrentPower = int(resp.registers[0]) -answer = '{"power":' + str(CurrentPower) + ',"powerc":' + str(TotalEnergy) + '} ' -f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') -json.dump(answer, f1) -f1.close() +answer = {"power": CurrentPower, "powerc": TotalEnergy} +with open(f'/var/www/html/openWB/ramdisk/smarthome_device_ret{devicenumber}', 'w') as f1: + json.dump(answer, f1) client.close() # clean disconnect from modbus server From b79316d13cf03fac8372f05f2742a7fdca7f14a2 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Mon, 22 Sep 2025 14:15:54 +0200 Subject: [PATCH 06/16] revert queue handler --- packages/helpermodules/logger.py | 58 +++++--------------------------- 1 file changed, 9 insertions(+), 49 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index 98069826cc..5882555666 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -1,9 +1,7 @@ import functools import logging -import logging.handlers from logging.handlers import RotatingFileHandler from pathlib import Path -import queue import sys import threading import typing_extensions @@ -210,70 +208,49 @@ def mb_to_bytes(megabytes: int) -> int: # to do: add smarthome and soc to in_memory_log_handlers, needs updates in individual thread calls # Main logger - log_queue = queue.Queue() - queue_handler = logging.handlers.QueueHandler(log_queue) main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5), backupCount=4) main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) main_file_handler.addFilter(RedactingFilter()) in_memory_log_handlers["main"] = InMemoryLogHandler(main_file_handler) in_memory_log_handlers["main"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) - logging.basicConfig(level=logging.DEBUG, handlers=[queue_handler, in_memory_log_handlers["main"]]) + logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler, in_memory_log_handlers["main"]]) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome")) - main_listener = logging.handlers.QueueListener(log_queue, main_file_handler) - main_listener.start() # Chargelog logger - chargelog_queue = queue.Queue() - chargelog_queue_handler = logging.handlers.QueueHandler(chargelog_queue) chargelog_log = logging.getLogger("chargelog") chargelog_log.propagate = False chargelog_file_handler = RotatingFileHandler( RAMDISK_PATH + 'chargelog.log', maxBytes=mb_to_bytes(2), backupCount=1) chargelog_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) chargelog_file_handler.addFilter(RedactingFilter()) - chargelog_log.addHandler(chargelog_queue_handler) - chargelog_listener = logging.handlers.QueueListener(chargelog_queue, chargelog_file_handler) - chargelog_listener.start() + chargelog_log.addHandler(chargelog_file_handler) # Data migration logger - data_migration_queue = queue.Queue() - data_migration_queue_handler = logging.handlers.QueueHandler(data_migration_queue) data_migration_log = logging.getLogger("data_migration") data_migration_log.propagate = False data_migration_file_handler = RotatingFileHandler( PERSISTENT_LOG_PATH + 'data_migration.log', maxBytes=mb_to_bytes(1), backupCount=1) data_migration_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) data_migration_file_handler.addFilter(RedactingFilter()) - data_migration_log.addHandler(data_migration_queue_handler) - data_migration_listener = logging.handlers.QueueListener(data_migration_queue, data_migration_file_handler) - data_migration_listener.start() + data_migration_log.addHandler(data_migration_file_handler) # MQTT logger - mqtt_queue = queue.Queue() - mqtt_queue_handler = logging.handlers.QueueHandler(mqtt_queue) mqtt_log = logging.getLogger("mqtt") mqtt_log.propagate = False mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1) mqtt_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) mqtt_file_handler.addFilter(RedactingFilter()) - mqtt_log.addHandler(mqtt_queue_handler) - mqtt_listener = logging.handlers.QueueListener(mqtt_queue, mqtt_file_handler) - mqtt_listener.start() + mqtt_log.addHandler(mqtt_file_handler) # Steuve control command logger - steuve_control_command_queue = queue.Queue() - steuve_control_command_queue_handler = logging.handlers.QueueHandler(steuve_control_command_queue) steuve_control_command_log = logging.getLogger("steuve_control_command") steuve_control_command_log.propagate = False steuve_control_command_file_handler = RotatingFileHandler( PERSISTENT_LOG_PATH + 'steuve_control_command.log', maxBytes=mb_to_bytes(80), backupCount=1) steuve_control_command_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) - steuve_control_command_log.addHandler(steuve_control_command_queue_handler) - steuve_control_command_listener = logging.handlers.QueueListener(steuve_control_command_queue, - steuve_control_command_file_handler) - steuve_control_command_listener.start() + steuve_control_command_log.addHandler(steuve_control_command_file_handler) # Garbage collector logger garbage_collector_queue = queue.Queue() @@ -302,33 +279,23 @@ def mb_to_bytes(megabytes: int) -> int: tracemalloc_listener.start() # Smarthome logger - smarthome_queue = queue.Queue() - smarthome_queue_handler = logging.handlers.QueueHandler(smarthome_queue) smarthome_log_handler = RotatingFileHandler(RAMDISK_PATH + 'smarthome.log', maxBytes=mb_to_bytes(1), backupCount=1) smarthome_log_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) smarthome_log_handler.addFilter(functools.partial(filter_pos, "smarthome")) smarthome_log_handler.addFilter(RedactingFilter()) - logging.getLogger().addHandler(smarthome_queue_handler) - smarthome_listener = logging.handlers.QueueListener(smarthome_queue, smarthome_log_handler) - smarthome_listener.start() + logging.getLogger().addHandler(smarthome_log_handler) # SoC logger - soc_queue = queue.Queue() - soc_queue_handler = logging.handlers.QueueHandler(soc_queue) soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) soc_log_handler.addFilter(functools.partial(filter_pos, "soc")) soc_log_handler.addFilter(RedactingFilter()) in_memory_log_handlers["soc"] = InMemoryLogHandler(soc_log_handler) in_memory_log_handlers["soc"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) - logging.getLogger().addHandler(soc_queue_handler) + logging.getLogger().addHandler(soc_log_handler) logging.getLogger().addHandler(in_memory_log_handlers["soc"]) - soc_listener = logging.handlers.QueueListener(soc_queue, soc_log_handler) - soc_listener.start() # Internal chargepoint logger - internal_chargepoint_queue = queue.Queue() - internal_chargepoint_queue_handler = logging.handlers.QueueHandler(internal_chargepoint_queue) internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log', maxBytes=mb_to_bytes(1), backupCount=1) @@ -337,24 +304,17 @@ def mb_to_bytes(megabytes: int) -> int: internal_chargepoint_log_handler.addFilter(RedactingFilter()) in_memory_log_handlers["internal_chargepoint"] = InMemoryLogHandler(internal_chargepoint_log_handler) in_memory_log_handlers["internal_chargepoint"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) - logging.getLogger().addHandler(internal_chargepoint_queue_handler) + logging.getLogger().addHandler(internal_chargepoint_log_handler) logging.getLogger().addHandler(in_memory_log_handlers["internal_chargepoint"]) - internal_chargepoint_listener = logging.handlers.QueueListener(internal_chargepoint_queue, - internal_chargepoint_log_handler) - internal_chargepoint_listener.start() # urllib3 logger - urllib3_queue = queue.Queue() - urllib3_queue_handler = logging.handlers.QueueHandler(urllib3_queue) urllib3_log = logging.getLogger("urllib3.connectionpool") urllib3_log.propagate = True urllib3_file_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) urllib3_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) urllib3_file_handler.addFilter(RedactingFilter()) urllib3_file_handler.addFilter(functools.partial(filter_pos, "soc")) - urllib3_log.addHandler(urllib3_queue_handler) - urllib3_listener = logging.handlers.QueueListener(urllib3_queue, urllib3_file_handler) - urllib3_listener.start() + urllib3_log.addHandler(urllib3_file_handler) logging.getLogger("pymodbus").setLevel(logging.WARNING) logging.getLogger("uModbus").setLevel(logging.WARNING) From 41dabdc3e9f2da58a9843a671b5956f9b90e3fc5 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Tue, 14 Oct 2025 09:29:56 +0200 Subject: [PATCH 07/16] Revert "Keep log messages from last full run and last run with error or warnings (#2118)" This reverts commit e201decd37629d5ccdb79b8654d310a624cc8779. --- packages/helpermodules/logger.py | 117 +------------------------------ packages/main.py | 5 -- packages/modules/update_soc.py | 4 -- 3 files changed, 2 insertions(+), 124 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index 5882555666..ea29e707bd 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -6,15 +6,11 @@ import threading import typing_extensions import re -import io -import os -import shutil FORMAT_STR_DETAILED = '%(asctime)s - {%(name)s:%(lineno)s} - {%(levelname)s:%(threadName)s} - %(message)s' FORMAT_STR_SHORT = '%(asctime)s - %(message)s' RAMDISK_PATH = str(Path(__file__).resolve().parents[2]) + '/ramdisk/' PERSISTENT_LOG_PATH = str(Path(__file__).resolve().parents[2]) + '/data/log/' -NUMBER_OF_LOGFILES = 3 KNOWN_SENSITIVE_FIELDS = [ 'password', 'secret', 'token', 'apikey', 'access_token', @@ -108,117 +104,18 @@ def filter_pos(name: str, record) -> bool: return False -class InMemoryLogHandler(logging.Handler): - def __init__(self, base_handler=None): - super().__init__() - self.base_handler = base_handler - self.log_stream = io.StringIO() - self.has_warning_or_error = False - - def emit(self, record): - if self.base_handler is None or self.base_handler.filter(record): - msg = self.format(record) - self.log_stream.write(msg + '\n') - if record.levelno >= logging.WARNING: - self.has_warning_or_error = True - - def get_logs(self): - return self.log_stream.getvalue() - - def clear(self): - self.log_stream = io.StringIO() - self.has_warning_or_error = False - - -def clear_in_memory_log_handler(logger_name: str = None) -> None: - global in_memory_log_handlers - if logger_name is None: - # Clear all in-memory log handlers - for handler in in_memory_log_handlers.values(): - handler.clear() - else: - # Clear specified in-memory log handler - if logger_name in in_memory_log_handlers: - in_memory_log_handlers[logger_name].clear() - - -def write_logs_to_file(logger_name: str = None) -> None: - global in_memory_log_handlers - - def rotate_logs(base_path: str, name: str): - # Rotate the log files - for i in range(NUMBER_OF_LOGFILES-1, 0, -1): - src = os.path.join(base_path, f'{name}.previous{i}.log') - dst = os.path.join(base_path, f'{name}.previous{i+1}.log') - if os.path.exists(src): - shutil.move(src, dst) - # Move the current log to previous1 - current_log = os.path.join(base_path, f'{name}.current.log') - if os.path.exists(current_log): - shutil.move(current_log, os.path.join(base_path, f'{name}.previous1.log')) - - def combine_logs(base_path: str, name: str): - latest_log_path = os.path.join(base_path, f'{name}.latest.log') - with open(latest_log_path, 'w') as latest_log: - for i in range(NUMBER_OF_LOGFILES-1, -1, -1): - log_file = os.path.join( - base_path, f'{name}.previous{i}.log') if i > 0 else os.path.join(base_path, f'{name}.current.log') - if os.path.exists(log_file): - with open(log_file, 'r') as f: - latest_log.write(f.read()) - - if logger_name is None: - # Write logs for all in-memory log handlers - for name, handler in in_memory_log_handlers.items(): - logs = handler.get_logs() - if logs: - rotate_logs(RAMDISK_PATH, name) - with open(os.path.join(RAMDISK_PATH, f'{name}.current.log'), 'w') as f: - f.write(logs) - combine_logs(RAMDISK_PATH, name) - - # If any warning or error messages were logged, create a -warning copy - if handler.has_warning_or_error: - with open(os.path.join(RAMDISK_PATH, f'{name}.latest-warning.log'), 'w') as f: - f.write(logs) - - else: - # Write logs for specified in-memory log handler - if logger_name in in_memory_log_handlers: - handler = in_memory_log_handlers[logger_name] - logs = handler.get_logs() - if logs: - rotate_logs(RAMDISK_PATH, logger_name) - with open(os.path.join(RAMDISK_PATH, f'{logger_name}.current.log'), 'w') as f: - f.write(logs) - combine_logs(RAMDISK_PATH, logger_name) - - # If any warning or error messages were logged, create a -warning copy - if handler.has_warning_or_error: - with open(os.path.join(RAMDISK_PATH, f'{logger_name}.latest-warning.log'), 'w') as f: - f.write(logs) - - def setup_logging() -> None: def mb_to_bytes(megabytes: int) -> int: return megabytes * 1000000 - - global in_memory_log_handlers - in_memory_log_handlers = {name: InMemoryLogHandler() for name in ["main", "internal_chargepoint"]} - # to do: add smarthome and soc to in_memory_log_handlers, needs updates in individual thread calls - - # Main logger + # Mehrere kleine Dateien verwenden, damit nicht zu viel verworfen wird, wenn die Datei voll ist. main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5), backupCount=4) main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) main_file_handler.addFilter(RedactingFilter()) - in_memory_log_handlers["main"] = InMemoryLogHandler(main_file_handler) - in_memory_log_handlers["main"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) - logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler, in_memory_log_handlers["main"]]) + logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler]) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome")) - # Chargelog logger chargelog_log = logging.getLogger("chargelog") chargelog_log.propagate = False chargelog_file_handler = RotatingFileHandler( @@ -227,7 +124,6 @@ def mb_to_bytes(megabytes: int) -> int: chargelog_file_handler.addFilter(RedactingFilter()) chargelog_log.addHandler(chargelog_file_handler) - # Data migration logger data_migration_log = logging.getLogger("data_migration") data_migration_log.propagate = False data_migration_file_handler = RotatingFileHandler( @@ -236,7 +132,6 @@ def mb_to_bytes(megabytes: int) -> int: data_migration_file_handler.addFilter(RedactingFilter()) data_migration_log.addHandler(data_migration_file_handler) - # MQTT logger mqtt_log = logging.getLogger("mqtt") mqtt_log.propagate = False mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1) @@ -285,27 +180,19 @@ def mb_to_bytes(megabytes: int) -> int: smarthome_log_handler.addFilter(RedactingFilter()) logging.getLogger().addHandler(smarthome_log_handler) - # SoC logger soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) soc_log_handler.addFilter(functools.partial(filter_pos, "soc")) soc_log_handler.addFilter(RedactingFilter()) - in_memory_log_handlers["soc"] = InMemoryLogHandler(soc_log_handler) - in_memory_log_handlers["soc"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) logging.getLogger().addHandler(soc_log_handler) - logging.getLogger().addHandler(in_memory_log_handlers["soc"]) - # Internal chargepoint logger internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log', maxBytes=mb_to_bytes(1), backupCount=1) internal_chargepoint_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint")) internal_chargepoint_log_handler.addFilter(RedactingFilter()) - in_memory_log_handlers["internal_chargepoint"] = InMemoryLogHandler(internal_chargepoint_log_handler) - in_memory_log_handlers["internal_chargepoint"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) logging.getLogger().addHandler(internal_chargepoint_log_handler) - logging.getLogger().addHandler(in_memory_log_handlers["internal_chargepoint"]) # urllib3 logger urllib3_log = logging.getLogger("urllib3.connectionpool") diff --git a/packages/main.py b/packages/main.py index 12fddb3891..d9574bb369 100755 --- a/packages/main.py +++ b/packages/main.py @@ -172,10 +172,6 @@ def handler_with_control_interval(): self.interval_counter = 1 else: self.interval_counter = self.interval_counter + 1 - - # In-Memory Log-Handler zurücksetzen - logger.clear_in_memory_log_handler("main") - log.info("# ***Start*** ") # log.debug(run_command.run_shell_command("top -b -n 1 | head -n 20")) # log.debug(f'Drosselung: {run_command.run_shell_command("if which vcgencmd >/dev/null; then vcgencmd get_throttled; else echo not found; fi")}') @@ -184,7 +180,6 @@ def handler_with_control_interval(): return try: handler_with_control_interval() - logger.write_logs_to_file("main") write_gc_stats() #print_active_threads_and_referrers() #analyze_function_origins() diff --git a/packages/modules/update_soc.py b/packages/modules/update_soc.py index 998f4caab7..1046b760d3 100644 --- a/packages/modules/update_soc.py +++ b/packages/modules/update_soc.py @@ -12,7 +12,6 @@ from helpermodules.utils import joined_thread_handler from modules.common.abstract_vehicle import VehicleUpdateData from modules.utils import wait_for_module_update_completed -from helpermodules.logger import clear_in_memory_log_handler, write_logs_to_file log = logging.getLogger(__name__) @@ -31,17 +30,14 @@ def update(self) -> None: self.event_update_soc.clear() topic = "openWB/set/vehicle/set/vehicle_update_completed" try: - clear_in_memory_log_handler("soc") threads_update, threads_store = self._get_threads() joined_thread_handler(threads_update, 300) wait_for_module_update_completed(self.event_vehicle_update_completed, topic) # threads_store = self._filter_failed_store_threads(threads_store) joined_thread_handler(threads_store, data.data.general_data.data.control_interval/3) wait_for_module_update_completed(self.event_vehicle_update_completed, topic) - write_logs_to_file("soc") except Exception: log.exception("Fehler im update_soc-Modul") - write_logs_to_file("soc") def _get_threads(self) -> Tuple[List[Thread], List[Thread]]: threads_update, threads_store = [], [] From 1b3335af8403047b751f060818df33d6dfd7951d Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Tue, 14 Oct 2025 09:33:01 +0200 Subject: [PATCH 08/16] fix --- packages/helpermodules/logger.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index ea29e707bd..5e56e5a891 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -148,30 +148,20 @@ def mb_to_bytes(megabytes: int) -> int: steuve_control_command_log.addHandler(steuve_control_command_file_handler) # Garbage collector logger - garbage_collector_queue = queue.Queue() - garbage_collector_queue_handler = logging.handlers.QueueHandler(garbage_collector_queue) garbage_collector_log = logging.getLogger("garbage_collector") garbage_collector_log.propagate = False garbage_collector_file_handler = RotatingFileHandler( RAMDISK_PATH + 'garbage_collector.log', maxBytes=mb_to_bytes(0.5), backupCount=1) garbage_collector_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) - garbage_collector_log.addHandler(garbage_collector_queue_handler) - garbage_collector_listener = logging.handlers.QueueListener(garbage_collector_queue, - garbage_collector_file_handler) - garbage_collector_listener.start() + garbage_collector_log.addHandler(garbage_collector_file_handler) # tracemalloc logger - tracemalloc_queue = queue.Queue() - tracemalloc_queue_handler = logging.handlers.QueueHandler(tracemalloc_queue) tracemalloc_log = logging.getLogger("tracemalloc") tracemalloc_log.propagate = False tracemalloc_file_handler = RotatingFileHandler( RAMDISK_PATH + 'tracemalloc.log', maxBytes=mb_to_bytes(0.5), backupCount=1) tracemalloc_file_handler.setFormatter(logging.Formatter(FORMAT_STR_SHORT)) - tracemalloc_log.addHandler(tracemalloc_queue_handler) - tracemalloc_listener = logging.handlers.QueueListener(tracemalloc_queue, - tracemalloc_file_handler) - tracemalloc_listener.start() + tracemalloc_log.addHandler(tracemalloc_file_handler) # Smarthome logger smarthome_log_handler = RotatingFileHandler(RAMDISK_PATH + 'smarthome.log', maxBytes=mb_to_bytes(1), backupCount=1) From 888c097bc33f707ceb616277d2203d10e988551f Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:08:49 +0200 Subject: [PATCH 09/16] Revert "Revert "Keep log messages from last full run and last run with error or warnings (#2118)"" This reverts commit 41dabdc3e9f2da58a9843a671b5956f9b90e3fc5. --- packages/helpermodules/logger.py | 117 ++++++++++++++++++++++++++++++- packages/main.py | 5 ++ packages/modules/update_soc.py | 4 ++ 3 files changed, 124 insertions(+), 2 deletions(-) diff --git a/packages/helpermodules/logger.py b/packages/helpermodules/logger.py index 5e56e5a891..436b8a6cc5 100644 --- a/packages/helpermodules/logger.py +++ b/packages/helpermodules/logger.py @@ -6,11 +6,15 @@ import threading import typing_extensions import re +import io +import os +import shutil FORMAT_STR_DETAILED = '%(asctime)s - {%(name)s:%(lineno)s} - {%(levelname)s:%(threadName)s} - %(message)s' FORMAT_STR_SHORT = '%(asctime)s - %(message)s' RAMDISK_PATH = str(Path(__file__).resolve().parents[2]) + '/ramdisk/' PERSISTENT_LOG_PATH = str(Path(__file__).resolve().parents[2]) + '/data/log/' +NUMBER_OF_LOGFILES = 3 KNOWN_SENSITIVE_FIELDS = [ 'password', 'secret', 'token', 'apikey', 'access_token', @@ -104,18 +108,117 @@ def filter_pos(name: str, record) -> bool: return False +class InMemoryLogHandler(logging.Handler): + def __init__(self, base_handler=None): + super().__init__() + self.base_handler = base_handler + self.log_stream = io.StringIO() + self.has_warning_or_error = False + + def emit(self, record): + if self.base_handler is None or self.base_handler.filter(record): + msg = self.format(record) + self.log_stream.write(msg + '\n') + if record.levelno >= logging.WARNING: + self.has_warning_or_error = True + + def get_logs(self): + return self.log_stream.getvalue() + + def clear(self): + self.log_stream = io.StringIO() + self.has_warning_or_error = False + + +def clear_in_memory_log_handler(logger_name: str = None) -> None: + global in_memory_log_handlers + if logger_name is None: + # Clear all in-memory log handlers + for handler in in_memory_log_handlers.values(): + handler.clear() + else: + # Clear specified in-memory log handler + if logger_name in in_memory_log_handlers: + in_memory_log_handlers[logger_name].clear() + + +def write_logs_to_file(logger_name: str = None) -> None: + global in_memory_log_handlers + + def rotate_logs(base_path: str, name: str): + # Rotate the log files + for i in range(NUMBER_OF_LOGFILES-1, 0, -1): + src = os.path.join(base_path, f'{name}.previous{i}.log') + dst = os.path.join(base_path, f'{name}.previous{i+1}.log') + if os.path.exists(src): + shutil.move(src, dst) + # Move the current log to previous1 + current_log = os.path.join(base_path, f'{name}.current.log') + if os.path.exists(current_log): + shutil.move(current_log, os.path.join(base_path, f'{name}.previous1.log')) + + def combine_logs(base_path: str, name: str): + latest_log_path = os.path.join(base_path, f'{name}.latest.log') + with open(latest_log_path, 'w') as latest_log: + for i in range(NUMBER_OF_LOGFILES-1, -1, -1): + log_file = os.path.join( + base_path, f'{name}.previous{i}.log') if i > 0 else os.path.join(base_path, f'{name}.current.log') + if os.path.exists(log_file): + with open(log_file, 'r') as f: + latest_log.write(f.read()) + + if logger_name is None: + # Write logs for all in-memory log handlers + for name, handler in in_memory_log_handlers.items(): + logs = handler.get_logs() + if logs: + rotate_logs(RAMDISK_PATH, name) + with open(os.path.join(RAMDISK_PATH, f'{name}.current.log'), 'w') as f: + f.write(logs) + combine_logs(RAMDISK_PATH, name) + + # If any warning or error messages were logged, create a -warning copy + if handler.has_warning_or_error: + with open(os.path.join(RAMDISK_PATH, f'{name}.latest-warning.log'), 'w') as f: + f.write(logs) + + else: + # Write logs for specified in-memory log handler + if logger_name in in_memory_log_handlers: + handler = in_memory_log_handlers[logger_name] + logs = handler.get_logs() + if logs: + rotate_logs(RAMDISK_PATH, logger_name) + with open(os.path.join(RAMDISK_PATH, f'{logger_name}.current.log'), 'w') as f: + f.write(logs) + combine_logs(RAMDISK_PATH, logger_name) + + # If any warning or error messages were logged, create a -warning copy + if handler.has_warning_or_error: + with open(os.path.join(RAMDISK_PATH, f'{logger_name}.latest-warning.log'), 'w') as f: + f.write(logs) + + def setup_logging() -> None: def mb_to_bytes(megabytes: int) -> int: return megabytes * 1000000 - # Mehrere kleine Dateien verwenden, damit nicht zu viel verworfen wird, wenn die Datei voll ist. + + global in_memory_log_handlers + in_memory_log_handlers = {name: InMemoryLogHandler() for name in ["main", "internal_chargepoint"]} + # to do: add smarthome and soc to in_memory_log_handlers, needs updates in individual thread calls + + # Main logger main_file_handler = RotatingFileHandler(RAMDISK_PATH + 'main.log', maxBytes=mb_to_bytes(5), backupCount=4) main_file_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) main_file_handler.addFilter(RedactingFilter()) - logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler]) + in_memory_log_handlers["main"] = InMemoryLogHandler(main_file_handler) + in_memory_log_handlers["main"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) + logging.basicConfig(level=logging.DEBUG, handlers=[main_file_handler, in_memory_log_handlers["main"]]) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "soc")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "Internal Chargepoint")) logging.getLogger().handlers[0].addFilter(functools.partial(filter_neg, "smarthome")) + # Chargelog logger chargelog_log = logging.getLogger("chargelog") chargelog_log.propagate = False chargelog_file_handler = RotatingFileHandler( @@ -124,6 +227,7 @@ def mb_to_bytes(megabytes: int) -> int: chargelog_file_handler.addFilter(RedactingFilter()) chargelog_log.addHandler(chargelog_file_handler) + # Data migration logger data_migration_log = logging.getLogger("data_migration") data_migration_log.propagate = False data_migration_file_handler = RotatingFileHandler( @@ -132,6 +236,7 @@ def mb_to_bytes(megabytes: int) -> int: data_migration_file_handler.addFilter(RedactingFilter()) data_migration_log.addHandler(data_migration_file_handler) + # MQTT logger mqtt_log = logging.getLogger("mqtt") mqtt_log.propagate = False mqtt_file_handler = RotatingFileHandler(RAMDISK_PATH + 'mqtt.log', maxBytes=mb_to_bytes(3), backupCount=1) @@ -170,19 +275,27 @@ def mb_to_bytes(megabytes: int) -> int: smarthome_log_handler.addFilter(RedactingFilter()) logging.getLogger().addHandler(smarthome_log_handler) + # SoC logger soc_log_handler = RotatingFileHandler(RAMDISK_PATH + 'soc.log', maxBytes=mb_to_bytes(2), backupCount=1) soc_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) soc_log_handler.addFilter(functools.partial(filter_pos, "soc")) soc_log_handler.addFilter(RedactingFilter()) + in_memory_log_handlers["soc"] = InMemoryLogHandler(soc_log_handler) + in_memory_log_handlers["soc"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) logging.getLogger().addHandler(soc_log_handler) + logging.getLogger().addHandler(in_memory_log_handlers["soc"]) + # Internal chargepoint logger internal_chargepoint_log_handler = RotatingFileHandler(RAMDISK_PATH + 'internal_chargepoint.log', maxBytes=mb_to_bytes(1), backupCount=1) internal_chargepoint_log_handler.setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) internal_chargepoint_log_handler.addFilter(functools.partial(filter_pos, "Internal Chargepoint")) internal_chargepoint_log_handler.addFilter(RedactingFilter()) + in_memory_log_handlers["internal_chargepoint"] = InMemoryLogHandler(internal_chargepoint_log_handler) + in_memory_log_handlers["internal_chargepoint"].setFormatter(logging.Formatter(FORMAT_STR_DETAILED)) logging.getLogger().addHandler(internal_chargepoint_log_handler) + logging.getLogger().addHandler(in_memory_log_handlers["internal_chargepoint"]) # urllib3 logger urllib3_log = logging.getLogger("urllib3.connectionpool") diff --git a/packages/main.py b/packages/main.py index d9574bb369..12fddb3891 100755 --- a/packages/main.py +++ b/packages/main.py @@ -172,6 +172,10 @@ def handler_with_control_interval(): self.interval_counter = 1 else: self.interval_counter = self.interval_counter + 1 + + # In-Memory Log-Handler zurücksetzen + logger.clear_in_memory_log_handler("main") + log.info("# ***Start*** ") # log.debug(run_command.run_shell_command("top -b -n 1 | head -n 20")) # log.debug(f'Drosselung: {run_command.run_shell_command("if which vcgencmd >/dev/null; then vcgencmd get_throttled; else echo not found; fi")}') @@ -180,6 +184,7 @@ def handler_with_control_interval(): return try: handler_with_control_interval() + logger.write_logs_to_file("main") write_gc_stats() #print_active_threads_and_referrers() #analyze_function_origins() diff --git a/packages/modules/update_soc.py b/packages/modules/update_soc.py index 1046b760d3..998f4caab7 100644 --- a/packages/modules/update_soc.py +++ b/packages/modules/update_soc.py @@ -12,6 +12,7 @@ from helpermodules.utils import joined_thread_handler from modules.common.abstract_vehicle import VehicleUpdateData from modules.utils import wait_for_module_update_completed +from helpermodules.logger import clear_in_memory_log_handler, write_logs_to_file log = logging.getLogger(__name__) @@ -30,14 +31,17 @@ def update(self) -> None: self.event_update_soc.clear() topic = "openWB/set/vehicle/set/vehicle_update_completed" try: + clear_in_memory_log_handler("soc") threads_update, threads_store = self._get_threads() joined_thread_handler(threads_update, 300) wait_for_module_update_completed(self.event_vehicle_update_completed, topic) # threads_store = self._filter_failed_store_threads(threads_store) joined_thread_handler(threads_store, data.data.general_data.data.control_interval/3) wait_for_module_update_completed(self.event_vehicle_update_completed, topic) + write_logs_to_file("soc") except Exception: log.exception("Fehler im update_soc-Modul") + write_logs_to_file("soc") def _get_threads(self) -> Tuple[List[Thread], List[Thread]]: threads_update, threads_store = [], [] From 15fc979cac33ce8b669427d275b292e6bf90a992 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:08:00 +0200 Subject: [PATCH 10/16] fix in-mem handler --- packages/main.py | 1 + .../internal_chargepoint_handler.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/packages/main.py b/packages/main.py index 12fddb3891..1bd12d4d4f 100755 --- a/packages/main.py +++ b/packages/main.py @@ -275,6 +275,7 @@ def handler_hour(self): for cp in data.data.cp_data.values(): calculate_charge_cost(cp) data.data.optional_data.et_get_prices() + logger.clear_in_memory_log_handler(None) except Exception: log.exception("Fehler im Main-Modul") diff --git a/packages/modules/internal_chargepoint_handler/internal_chargepoint_handler.py b/packages/modules/internal_chargepoint_handler/internal_chargepoint_handler.py index 535f7a1f9d..d9f19dfeae 100644 --- a/packages/modules/internal_chargepoint_handler/internal_chargepoint_handler.py +++ b/packages/modules/internal_chargepoint_handler/internal_chargepoint_handler.py @@ -7,6 +7,7 @@ from helpermodules import timecheck from helpermodules import pub +from helpermodules.logger import clear_in_memory_log_handler from helpermodules.pub import Pub, pub_single from helpermodules.subdata import SubData from modules.chargepoints.internal_openwb.config import InternalChargepointMode @@ -147,6 +148,7 @@ def _loop(): while True: if self.event_stop.is_set(): break + clear_in_memory_log_handler("internal_chargepoint") log.debug("***Start***") data = copy.deepcopy(SubData.internal_chargepoint_data) log.debug(data) From ef40ba3b50eb89783c3070d92065179e85865ae5 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:10:38 +0200 Subject: [PATCH 11/16] typo --- packages/modules/loadvars.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/modules/loadvars.py b/packages/modules/loadvars.py index e46d4e2150..ef1b07618a 100644 --- a/packages/modules/loadvars.py +++ b/packages/modules/loadvars.py @@ -51,8 +51,7 @@ def _set_values(self) -> List[str]: args=(), name=f"set values cp{cp.chargepoint_module.config.id}")) except Exception: log.exception(f"Fehler im loadvars-Modul bei Element {cp.num}") - result = joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) - return result + return joined_thread_handler(modules_threads, data.data.general_data.data.control_interval/3) def _update_values_of_level(self, elements, not_finished_threads: List[str]) -> None: """Threads, um von der niedrigsten Ebene der Hierarchie Werte ggf. miteinander zu verrechnen und zu From 22474c303122d21cab2c8b7a7d05dc9b33289df4 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:17:29 +0200 Subject: [PATCH 12/16] fix smarthome --- packages/modules/smarthome/fronius/watt.py | 3 +-- packages/modules/smarthome/json/watt.py | 3 +-- packages/modules/smarthome/mystrom/watt.py | 2 +- packages/modules/smarthome/nibe/watt.py | 3 --- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/modules/smarthome/fronius/watt.py b/packages/modules/smarthome/fronius/watt.py index 4470418ec8..9f805c35b8 100644 --- a/packages/modules/smarthome/fronius/watt.py +++ b/packages/modules/smarthome/fronius/watt.py @@ -30,5 +30,4 @@ except Exception: powerc = 0 -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +log.debug("Device" + str(devicenumber) + '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/json/watt.py b/packages/modules/smarthome/json/watt.py index 5e82995631..a5131ee715 100644 --- a/packages/modules/smarthome/json/watt.py +++ b/packages/modules/smarthome/json/watt.py @@ -29,5 +29,4 @@ except Exception: powerc = 0 -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +log.debug('Device' + str(devicenumber) + '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/mystrom/watt.py b/packages/modules/smarthome/mystrom/watt.py index 1a4faf97bf..e58764e96b 100644 --- a/packages/modules/smarthome/mystrom/watt.py +++ b/packages/modules/smarthome/mystrom/watt.py @@ -25,4 +25,4 @@ answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + \ str(relais) + ',"temp0":' + str(temp) + '} ' with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug(answer) + log.debug('Device' + str(devicenumber) + ' ' + answer) diff --git a/packages/modules/smarthome/nibe/watt.py b/packages/modules/smarthome/nibe/watt.py index 99ecca59a1..d723965d22 100644 --- a/packages/modules/smarthome/nibe/watt.py +++ b/packages/modules/smarthome/nibe/watt.py @@ -1,11 +1,8 @@ #!/usr/bin/python3 -import logging import sys import json from pymodbus.client.sync import ModbusTcpClient -log = logging.getLogger(__name__) - # get variables from arguments devicenumber = str(sys.argv[1]) # SmartHome device number SERVER_HOST = str(sys.argv[2]) # IP of server to connect to From 0740c9fa2e0795cfb902ebfc9fdfa17a37ca20a0 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:21:14 +0200 Subject: [PATCH 13/16] Revert "fix smarthome" This reverts commit 22474c303122d21cab2c8b7a7d05dc9b33289df4. --- packages/modules/smarthome/fronius/watt.py | 3 ++- packages/modules/smarthome/json/watt.py | 3 ++- packages/modules/smarthome/mystrom/watt.py | 2 +- packages/modules/smarthome/nibe/watt.py | 3 +++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/modules/smarthome/fronius/watt.py b/packages/modules/smarthome/fronius/watt.py index 9f805c35b8..4470418ec8 100644 --- a/packages/modules/smarthome/fronius/watt.py +++ b/packages/modules/smarthome/fronius/watt.py @@ -30,4 +30,5 @@ except Exception: powerc = 0 -log.debug("Device" + str(devicenumber) + '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/json/watt.py b/packages/modules/smarthome/json/watt.py index a5131ee715..5e82995631 100644 --- a/packages/modules/smarthome/json/watt.py +++ b/packages/modules/smarthome/json/watt.py @@ -29,4 +29,5 @@ except Exception: powerc = 0 -log.debug('Device' + str(devicenumber) + '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: + log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') diff --git a/packages/modules/smarthome/mystrom/watt.py b/packages/modules/smarthome/mystrom/watt.py index e58764e96b..1a4faf97bf 100644 --- a/packages/modules/smarthome/mystrom/watt.py +++ b/packages/modules/smarthome/mystrom/watt.py @@ -25,4 +25,4 @@ answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + \ str(relais) + ',"temp0":' + str(temp) + '} ' with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug('Device' + str(devicenumber) + ' ' + answer) + log.debug(answer) diff --git a/packages/modules/smarthome/nibe/watt.py b/packages/modules/smarthome/nibe/watt.py index d723965d22..99ecca59a1 100644 --- a/packages/modules/smarthome/nibe/watt.py +++ b/packages/modules/smarthome/nibe/watt.py @@ -1,8 +1,11 @@ #!/usr/bin/python3 +import logging import sys import json from pymodbus.client.sync import ModbusTcpClient +log = logging.getLogger(__name__) + # get variables from arguments devicenumber = str(sys.argv[1]) # SmartHome device number SERVER_HOST = str(sys.argv[2]) # IP of server to connect to From 9dac7b78c097fcd8c385e92266f27660d88aa435 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 09:21:22 +0200 Subject: [PATCH 14/16] Revert "use filehandler for smarthome" This reverts commit d77ef5141b36d24a5cf0d1246e004c3ecc90d348. --- .../smarthome/avmhomeautomation/avmcommon.py | 38 ++++++++++--- packages/modules/smarthome/fronius/watt.py | 9 ++- packages/modules/smarthome/json/watt.py | 9 ++- packages/modules/smarthome/mqtt/off.py | 5 +- packages/modules/smarthome/mqtt/on.py | 5 +- packages/modules/smarthome/mqtt/watt.py | 5 +- packages/modules/smarthome/mystrom/watt.py | 8 +-- packages/modules/smarthome/nibe/watt.py | 8 +-- packages/modules/smarthome/smaem/watt.py | 55 +++++++++++-------- packages/modules/smarthome/tasmota/watt.py | 5 +- packages/modules/smarthome/viessmann/off.py | 26 ++++++--- packages/modules/smarthome/viessmann/on.py | 27 ++++++--- packages/modules/smarthome/viessmann/watt.py | 22 ++++---- packages/modules/smarthome/we514/watt.py | 7 ++- 14 files changed, 141 insertions(+), 88 deletions(-) diff --git a/packages/modules/smarthome/avmhomeautomation/avmcommon.py b/packages/modules/smarthome/avmhomeautomation/avmcommon.py index 4dd27c5c94..8124f3d2c1 100644 --- a/packages/modules/smarthome/avmhomeautomation/avmcommon.py +++ b/packages/modules/smarthome/avmhomeautomation/avmcommon.py @@ -42,8 +42,9 @@ def __init__(self): if os.path.isfile(CACHEFILE): self.logMessage(LOGLEVELDEBUG, "found an AVM cache file, trying to load") try: - with open(CACHEFILE, 'r') as f: - self.cache = json.loads(f.read().strip()) + f = open(CACHEFILE, 'r') + self.cache = json.loads(f.read().strip()) + f.close() except Exception as e: self.logMessage(LOGLEVELDEBUG, "unable to load cache file: %s" % (e)) @@ -64,8 +65,9 @@ def writeCacheToRamdisk(self): else: cacheToWrite[login] = self.cache[login] try: - with open(CACHEFILE, 'w') as f: - json.dump(cacheToWrite, f) + f = open(CACHEFILE, 'w') + json.dump(cacheToWrite, f) + f.close() except Exception as e: self.logMessage(LOGLEVELDEBUG, "unable to write cache file: %s" % (e)) @@ -164,7 +166,28 @@ def connect(self): # logMessage writes a message to the logfile for the smarthome device. def logMessage(self, level, message): - log.debug(f'({self.devicenumber}) AVM (actor: {self.switchname}) {message}') + if level < self.loglevel: + return + now = time.localtime() # getstruct_time + time_string = time.strftime("%Y-%m-%d %H:%M:%S", now) + logfile_string = '/var/www/html/openWB/ramdisk/smarthome.log' + try: + if os.path.isfile(logfile_string): + f = open(logfile_string, 'a', encoding='utf8') + else: + f = open(logfile_string, 'w', encoding='utf8') + prefix = "" + if level == LOGLEVELDEBUG: + prefix = "[DEBUG] " + if level == LOGLEVELINFO: + prefix = "[INFO] " + if level == LOGLEVELERROR: + prefix = "[ERROR] " + log.debug('%s: (%s) AVM (actor: %s) %s%s' % + (time_string, self.devicenumber, self.switchname, prefix, message), file=f) + f.close() + except IOError: + pass # getDevicesDict returns a dictionary that maps defined actor names to its # unique hardware ID (called "AIN": "Actuator Identification Number") and @@ -339,8 +362,9 @@ def getActualPower(self): outFileString = '/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(self.devicenumber) self.logMessage(LOGLEVELDEBUG, "handing answer back to smarthomehandler via %s" % (outFileString)) try: - with open(outFileString, 'w') as f1: - json.dump(answer, f1) + f1 = open(outFileString, 'w') + json.dump(answer, f1) + f1.close() except IOError as e: self.logMessage(LOGLEVELERROR, "error writing power result %s" % (e)) log.debug(answer) # dump answer to stdout if file cannot be written diff --git a/packages/modules/smarthome/fronius/watt.py b/packages/modules/smarthome/fronius/watt.py index 4470418ec8..ca0eb4e505 100644 --- a/packages/modules/smarthome/fronius/watt.py +++ b/packages/modules/smarthome/fronius/watt.py @@ -1,12 +1,9 @@ #!/usr/bin/python3 -import logging import sys import json import jq import urllib.request -log = logging.getLogger(__name__) - devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) # IP-ADresse des Fronius Wechselrichters, mit dem der Zähler kommuniziert smid = int(sys.argv[3]) # ID des Zählers im Wechselrichter (Hauptzähler 0, weitere fortlaufend) @@ -30,5 +27,7 @@ except Exception: powerc = 0 -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +answer = '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}' +json.dump(answer, f1) +f1.close() diff --git a/packages/modules/smarthome/json/watt.py b/packages/modules/smarthome/json/watt.py index 5e82995631..efe2381ad9 100644 --- a/packages/modules/smarthome/json/watt.py +++ b/packages/modules/smarthome/json/watt.py @@ -1,12 +1,9 @@ #!/usr/bin/python3 -import logging import sys import json import jq import urllib.request -log = logging.getLogger(__name__) - devicenumber = str(sys.argv[1]) # Abfrage-URL, die die .json Antwort liefert. Z.B. # "http://192.168.0.150/solar_api/v1/GetMeterRealtimeData.cgi?Scope=Device&DeviceID=1" @@ -29,5 +26,7 @@ except Exception: powerc = 0 -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug('{"power":' + str(power) + ',"powerc":' + str(powerc) + '}') +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +answer = '{"power":' + str(power) + ',"powerc":' + str(powerc) + '}' +json.dump(answer, f1) +f1.close() diff --git a/packages/modules/smarthome/mqtt/off.py b/packages/modules/smarthome/mqtt/off.py index 672f1e9c28..d7cea04010 100644 --- a/packages/modules/smarthome/mqtt/off.py +++ b/packages/modules/smarthome/mqtt/off.py @@ -35,5 +35,6 @@ def on_message(client, userdata, msg) -> None: client.disconnect() file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' pvmodus = 0 -with open(file_stringpv, 'w') as f: - f.write(str(pvmodus)) +f = open(file_stringpv, 'w') +f.write(str(pvmodus)) +f.close() diff --git a/packages/modules/smarthome/mqtt/on.py b/packages/modules/smarthome/mqtt/on.py index 781d850a81..b5f70ed825 100644 --- a/packages/modules/smarthome/mqtt/on.py +++ b/packages/modules/smarthome/mqtt/on.py @@ -34,5 +34,6 @@ def on_message(client, userdata, msg) -> None: client.loop(timeout=2.0) client.disconnect() file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' -with open(file_stringpv, 'w') as f: - f.write(str(1)) +f = open(file_stringpv, 'w') +f.write(str(1)) +f.close() diff --git a/packages/modules/smarthome/mqtt/watt.py b/packages/modules/smarthome/mqtt/watt.py index d1cb945fbc..d590c36417 100644 --- a/packages/modules/smarthome/mqtt/watt.py +++ b/packages/modules/smarthome/mqtt/watt.py @@ -69,8 +69,9 @@ def on_message(client, userdata, msg) -> None: # PV-Modus pvmodus = 0 if os.path.isfile(file_stringpv): - with open(file_stringpv, 'r') as f: - pvmodus = int(f.read()) + f = open(file_stringpv, 'r') + pvmodus = int(f.read()) + f.close() answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) answer += ',"on":' + str(pvmodus) + ',"temp0":' + str(tempa) answer += ',"temp1":' + str(tempb) + ',"temp2":' + str(tempc) + '}' diff --git a/packages/modules/smarthome/mystrom/watt.py b/packages/modules/smarthome/mystrom/watt.py index 1a4faf97bf..3ae83cf393 100644 --- a/packages/modules/smarthome/mystrom/watt.py +++ b/packages/modules/smarthome/mystrom/watt.py @@ -1,12 +1,9 @@ #!/usr/bin/python3 -import logging import sys import time import json import urllib.request -log = logging.getLogger(__name__) - named_tuple = time.localtime() # getstruct_time time_string = time.strftime("%m/%d/%Y, %H:%M:%S mystrom watty.py", named_tuple) devicenumber = str(sys.argv[1]) @@ -24,5 +21,6 @@ powerc = 0 answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + \ str(relais) + ',"temp0":' + str(temp) + '} ' -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - log.debug(answer) +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +json.dump(answer, f1) +f1.close() diff --git a/packages/modules/smarthome/nibe/watt.py b/packages/modules/smarthome/nibe/watt.py index 99ecca59a1..3b9bc3f286 100644 --- a/packages/modules/smarthome/nibe/watt.py +++ b/packages/modules/smarthome/nibe/watt.py @@ -1,11 +1,8 @@ #!/usr/bin/python3 -import logging import sys import json from pymodbus.client.sync import ModbusTcpClient -log = logging.getLogger(__name__) - # get variables from arguments devicenumber = str(sys.argv[1]) # SmartHome device number SERVER_HOST = str(sys.argv[2]) # IP of server to connect to @@ -32,7 +29,8 @@ CurrentPower = None # Handle error case answer = '{"power":' + str(CurrentPower) + '}' -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - json.dump(answer, f1) +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +json.dump(answer, f1) +f1.close() client.close() # clean disconnect from modbus server diff --git a/packages/modules/smarthome/smaem/watt.py b/packages/modules/smarthome/smaem/watt.py index f31ef9c3d2..bfb767a89a 100644 --- a/packages/modules/smarthome/smaem/watt.py +++ b/packages/modules/smarthome/smaem/watt.py @@ -33,6 +33,7 @@ import os import datetime import time +import json import signal import socket import struct @@ -67,7 +68,7 @@ def abortprogram(signal, frame): timefile = '/var/www/html/openWB/ramdisk/smarthome_device_ret' + \ str(devicenumber) + '_time' # Dummy file needed for timestamp of last metering # Logfile for additional output beside of the smarthome.log - +debugfile = open('/var/www/html/openWB/ramdisk/smaem.log', 'a', newline='\r\n') sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -107,9 +108,9 @@ def abortprogram(signal, frame): "HomeManager (2) is sending in the network.") emparts = decode_speedwire(sock_data) -log.debug('SMA:: smaserial: #' + str(smaserial) + '# - Current SMA serial number:#' + - str(emparts['serial']) + '# - watt:#' + str(int(emparts.get("pconsume"))) + '# - wattc:#' + - str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) + '#') +debugfile.write(str(datetime.datetime.now()) + ': smaserial: #' + str(smaserial) + '# - Current SMA serial number:#' + + str(emparts['serial']) + '# - watt:#' + str(int(emparts.get("pconsume"))) + '# - wattc:#' + + str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) + '#\n') # Remember: We assume that beside of our EnergyMeter there are more SMA devices present (like HomeManager 2.0 or other # EnergyMeter) - so must not accept any data or smaserial = None @@ -117,8 +118,8 @@ def abortprogram(signal, frame): # our output variables watt = str(int(emparts.get("pconsume"))) wattc = str("{:.3f}".format(int(emparts.get('pconsumecounter')*1000))) - log.debug('SMA:: 1 - Our SMA EM ' + str(smaserial) + - ' is sending, everything fine. watt: #' + str(watt) + '# - wattc: #' + str(wattc) + '#') + debugfile.write(str(datetime.datetime.now()) + ': 1 - Our SMA EM ' + str(smaserial) + + ' is sending, everything fine. watt: #' + str(watt) + '# - wattc: #' + str(wattc) + '#\n') # Scenario 2: Our EnergyMeter is not sending but we have a returnfile which is older than n seconds (parameter # secondssincelastmetering) elif ((os.path.isfile(returnfile)) and @@ -127,27 +128,29 @@ def abortprogram(signal, frame): # We set "0" as current Power Consume (pconsume) and (from the existing ret-file) the last value for the Power # Consume Counter (pconsumecounter) watt = '0' - with open(returnfile, 'r') as ret: - lastvalues = ret.read() + ret = open(returnfile, 'r') + lastvalues = ret.read() + ret.close() timesincelastmetering = int(round(time.time(), 0)-lastmodificationtime) wattc = lastvalues[int(lastvalues.rfind('powerc')) + 9:lastvalues.find('}')] - log.debug('SMA:: 2 - Debug: time.time(): #' + str(round(time.time(), 0)) + - '# - lastmodificationtime: #' + str(lastmodificationtime) + - '# - timesincelastmetering: #' + str(timesincelastmetering) + - '# - int(secondssincelastmetering): #' + str(int(secondssincelastmetering)) + '#') - log.debug('SMA:: 2 - We create a fake ret-file. watt: #' + str(watt) + - '# - wattc: #' + str( + debugfile.write(str(datetime.datetime.now()) + ': 2 - Debug: time.time(): #' + str(round(time.time(), 0)) + + '# - lastmodificationtime: #' + str(lastmodificationtime) + + '# - timesincelastmetering: #' + str(timesincelastmetering) + + '# - int(secondssincelastmetering): #' + str(int(secondssincelastmetering)) + '#\n') + debugfile.write(str(datetime.datetime.now()) + ': 2 - We create a fake ret-file. watt: #' + str(watt) + + '# - wattc: #' + str( wattc) + '# - lastvalues: #' + lastvalues + '# - int-lastvalues.rfind-powerc: #' + - str((lastvalues.rfind('powerc'))) + '#') + str((lastvalues.rfind('powerc'))) + '#\n') # Scenario 3: Our EnergyMeter is not sending but we have a returnfile which is younger than n seconds # (parameter secondssincelastmetering) elif ((os.path.isfile(returnfile)) and (int((round(time.time(), 0)-lastmodificationtime)) < int(secondssincelastmetering))): # We have a ret-file which is younger than n seconds. We do nothing as the existing ret-file is good enough. - log.debug('SMA:: 3 - The existing ret-file is fine enough. round(time.time(),0): #' + str(round(time.time(), 0)) - + '# - lastmodificationtime: #' + str(lastmodificationtime) + '# - secondssincelastmetering: #' - + str(secondssincelastmetering) + '#') + debugfile.write(str(datetime.datetime.now()) + + ': 3 - The existing ret-file is fine enough. round(time.time(),0): #' + str(round(time.time(), 0)) + + '# - lastmodificationtime: #' + str(lastmodificationtime) + '# - secondssincelastmetering: #' + + str(secondssincelastmetering) + '#\n') sys.exit("Module SMAEM: No data received but we have historical data which is younger than " + str(secondssincelastmetering) + " seconds.") else: @@ -159,16 +162,20 @@ def abortprogram(signal, frame): # Module SMAEM: No data received and no historical data since boot time # Leistungsmessung smaem [...] Fehlermeldung: [Errno 2] No such file or directory: # '/var/www/html/openWB/ramdisk/smarthome_device_ret1' - log.debug('SMA:: 4 - No data received and no historical data since boottime') + debugfile.write(str(datetime.datetime.now()) + ': 4 - No data received and no historical data since boottime\n') sys.exit(str(datetime.datetime.now()) + ": Module SMAEM: No data received and no historical data since boottime") # General output section answer = '{"power":' + watt + ',"powerc":' + wattc + '}' +f = open(returnfile, 'w') +json.dump(answer, f) +f.close() -with open(timefile, 'w') as t: - t.write(str(datetime.datetime.now()) + - ': File is created by Smarthome module SMAEM for validating ' - 'the timestamp of the last return-file creation.') +t = open(timefile, 'w') +t.write(str(datetime.datetime.now()) + + ': File is created by Smarthome module SMAEM for validating the timestamp of the last return-file creation.') +t.close() -log.debug('SMA:: 99 - Output answer: #' + answer + '#') +debugfile.write(str(datetime.datetime.now()) + ': 99 - Output answer: #' + answer + '#\n') +debugfile.close() diff --git a/packages/modules/smarthome/tasmota/watt.py b/packages/modules/smarthome/tasmota/watt.py index 533d796d39..b5f9a54832 100644 --- a/packages/modules/smarthome/tasmota/watt.py +++ b/packages/modules/smarthome/tasmota/watt.py @@ -26,5 +26,6 @@ relais = 1 powerc = 0 answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + str(relais) + '} ' -with open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') as f1: - json.dump(answer, f1) +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +json.dump(answer, f1) +f1.close() diff --git a/packages/modules/smarthome/viessmann/off.py b/packages/modules/smarthome/viessmann/off.py index d294441d9e..12cf698ef9 100644 --- a/packages/modules/smarthome/viessmann/off.py +++ b/packages/modules/smarthome/viessmann/off.py @@ -1,10 +1,14 @@ #!/usr/bin/python3 import sys +import os +import time from pymodbus.client.sync import ModbusTcpClient import logging log = logging.getLogger(__name__) +named_tuple = time.localtime() # getstruct_time +time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann off.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) @@ -18,14 +22,22 @@ # coils read write boolean # register start 00000 # -log.debug(f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " - f"ueberschuss {uberschuss:6d} try to connect (modbus)") +file_string = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_viessmann.log' +file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' +if os.path.isfile(file_string): + f = open(file_string, 'a') +else: + f = open(file_string, 'w') +log.debug('%s devicenr %s ipadr %s ueberschuss %6d try to connect (modbus)' % + (time_string, devicenumber, ipadr, uberschuss), file=f) client = ModbusTcpClient(ipadr, port=502) rq = client.write_coil(16, False, unit=1) -log.debug(f"[Viessmann {devicenumber}] Modbus write_coil response: {rq}") +log.debug(rq, file=f) client.close() -log.debug( - f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " - "Einmalige Warmwasseraufbereitung deaktiviert CO-17 = 0") +log.debug('%s devicenr %s ipadr %s Einmalige Warmwasseraufbereitung deaktiviert CO-17 = 0 ' % + (time_string, devicenumber, ipadr), file=f) +f.close() pvmodus = 0 -log.debug(f"[Viessmann {devicenumber}] PV-Modus gesetzt: {pvmodus}") +f = open(file_stringpv, 'w') +f.write(str(pvmodus)) +f.close() diff --git a/packages/modules/smarthome/viessmann/on.py b/packages/modules/smarthome/viessmann/on.py index 5f7415cd16..f85a7d6d6f 100644 --- a/packages/modules/smarthome/viessmann/on.py +++ b/packages/modules/smarthome/viessmann/on.py @@ -1,10 +1,14 @@ #!/usr/bin/python3 import sys +import os +import time from pymodbus.client.sync import ModbusTcpClient import logging log = logging.getLogger(__name__) +named_tuple = time.localtime() # getstruct_time +time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann on.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) @@ -20,14 +24,21 @@ # coils read write boolean # register start 00000 # -log.debug( - f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " - f"ueberschuss {uberschuss:6d} try to connect (modbus)") +file_string = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_viessmann.log' +file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' +if os.path.isfile(file_string): + f = open(file_string, 'a') +else: + f = open(file_string, 'w') +log.debug('%s devicenr %s ipadr %s ueberschuss %6d try to connect (modbus)' % + (time_string, devicenumber, ipadr, uberschuss), file=f) client = ModbusTcpClient(ipadr, port=502) rq = client.write_coil(16, True, unit=1) -log.debug(f"[Viessmann {devicenumber}] Modbus write_coil response: {rq}") +log.debug(rq, file=f) client.close() -log.debug( - f"[Viessmann {devicenumber}] devicenr {devicenumber} ipadr {ipadr} " - "Einmalige Warmwasseraufbereitung aktiviert CO-17 = 1") -log.debug(f"[Viessmann {devicenumber}] PV-Modus gesetzt: 1") +log.debug('%s devicenr %s ipadr %s Einmalige Warmwasseraufbereitung aktiviert CO-17 = 1' % + (time_string, devicenumber, ipadr), file=f) +f.close() +f = open(file_stringpv, 'w') +f.write(str(1)) +f.close() diff --git a/packages/modules/smarthome/viessmann/watt.py b/packages/modules/smarthome/viessmann/watt.py index 5d192050aa..e46ec14986 100644 --- a/packages/modules/smarthome/viessmann/watt.py +++ b/packages/modules/smarthome/viessmann/watt.py @@ -1,24 +1,24 @@ #!/usr/bin/python3 - import sys import os +import time import json -import logging - -log = logging.getLogger(__name__) +named_tuple = time.localtime() # getstruct_time +time_string = time.strftime("%m/%d/%Y, %H:%M:%S viessmann watty.py", named_tuple) devicenumber = str(sys.argv[1]) ipadr = str(sys.argv[2]) uberschuss = int(sys.argv[3]) -file_stringpv = f'/var/www/html/openWB/ramdisk/smarthome_device_{devicenumber}_pv' +file_stringpv = '/var/www/html/openWB/ramdisk/smarthome_device_' + str(devicenumber) + '_pv' # PV-Modus pvmodus = 0 if os.path.isfile(file_stringpv): - with open(file_stringpv, 'r') as f: - pvmodus = int(f.read()) + f = open(file_stringpv, 'r') + pvmodus = int(f.read()) + f.close() aktpower = 0 powerc = 0 -answer = {"power": aktpower, "powerc": powerc, "on": pvmodus} -outfile = f'/var/www/html/openWB/ramdisk/smarthome_device_ret{devicenumber}' -with open(outfile, 'w') as f1: - json.dump(answer, f1) +answer = '{"power":' + str(aktpower) + ',"powerc":' + str(powerc) + ',"on":' + str(pvmodus) + '} ' +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +json.dump(answer, f1) +f1.close() diff --git a/packages/modules/smarthome/we514/watt.py b/packages/modules/smarthome/we514/watt.py index f9d3afe00e..5a12363c25 100644 --- a/packages/modules/smarthome/we514/watt.py +++ b/packages/modules/smarthome/we514/watt.py @@ -33,8 +33,9 @@ resp = client.read_holding_registers(CurrentPowerRegisterAddress, 1, unit=MODBUS_DEVICEID) CurrentPower = int(resp.registers[0]) -answer = {"power": CurrentPower, "powerc": TotalEnergy} -with open(f'/var/www/html/openWB/ramdisk/smarthome_device_ret{devicenumber}', 'w') as f1: - json.dump(answer, f1) +answer = '{"power":' + str(CurrentPower) + ',"powerc":' + str(TotalEnergy) + '} ' +f1 = open('/var/www/html/openWB/ramdisk/smarthome_device_ret' + str(devicenumber), 'w') +json.dump(answer, f1) +f1.close() client.close() # clean disconnect from modbus server From 11d60bc11abb2f362d94c1b0a9a816d57bbbdd73 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 10:47:03 +0200 Subject: [PATCH 15/16] remove tracemalloc --- packages/main.py | 96 ------------------------------------------------ 1 file changed, 96 deletions(-) diff --git a/packages/main.py b/packages/main.py index 1bd12d4d4f..176c869318 100755 --- a/packages/main.py +++ b/packages/main.py @@ -8,29 +8,10 @@ import threading import sys import functools -import gc # GC-Modul importieren -import collections -import tracemalloc # als erstes logging initialisieren, damit auch ImportError geloggt werden logger.setup_logging() log = logging.getLogger() -# GC-Logger einrichten -gc_logger = logging.getLogger("garbage_collector") -tracemalloc_logger = logging.getLogger("tracemalloc") -def gc_callback(phase, info): - objs = gc.get_objects() - type_counter = collections.Counter(type(obj).__name__ for obj in objs) - most_common = type_counter.most_common(10) - stats = { - "garbage": len(gc.garbage), - "objects": len(objs), - "counts": gc.get_count(), - "top_types": most_common, - } - gc_logger.error(f"GC-Phase: {phase}, Info: {info}, Stats: {stats}") - - from pathlib import Path from random import randrange import schedule @@ -185,10 +166,6 @@ def handler_with_control_interval(): try: handler_with_control_interval() logger.write_logs_to_file("main") - write_gc_stats() - #print_active_threads_and_referrers() - #analyze_function_origins() - log_memory_usage() finally: self.__release_lock("handler10Sec") except Exception: @@ -240,7 +217,6 @@ def handler5Min(self): general_internal_chargepoint_handler.internal_chargepoint_handler.heartbeat = False with ChangedValuesContext(loadvars_.event_module_update_completed): sub.system_data["system"].update_ip_address() - log_memory_usage(always_log=True) except Exception: log.exception("Fehler im Main-Modul") @@ -280,75 +256,6 @@ def handler_hour(self): log.exception("Fehler im Main-Modul") -old_memory_usage: int - -def log_memory_usage(always_log=False): - global old_memory_usage - with open("/proc/self/status") as f: - for line in f: - if line.startswith("VmRSS:"): - mem_kb = int(line.split()[1]) - memory_usage = mem_kb / 1024 # MB - if old_memory_usage + 30 < memory_usage or always_log: - tracemalloc_logger.error(f"Speicherverbrauch: {memory_usage:.2f} MB, vorheriger: {old_memory_usage:.2f} MB") - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('lineno') - for stat in top_stats[:10]: - tracemalloc_logger.error(stat) - old_memory_usage = memory_usage - -def write_gc_stats(): - """Schreibt GC-Statistiken in eine Datei über den eigenen Logger.""" - try: - objs = gc.get_objects() - type_counter = collections.Counter(type(obj).__name__ for obj in objs) - most_common = type_counter.most_common(10) - stats = { - "garbage": len(gc.garbage), - "objects": len(objs), - "counts": gc.get_count(), - "top_types": most_common, - } - gc_logger.error(stats) - except Exception: - log.exception("Fehler beim Schreiben der GC-Statistiken") - -# def analyze_function_origins(): -# """Analysiert, woher die Funktionsobjekte im Speicher stammen.""" -# import types -# gc.collect() -# objs = gc.get_objects() -# origins = {} -# for obj in objs: -# if isinstance(obj, types.FunctionType): -# try: -# key = ( -# getattr(obj, "__module__", None), -# getattr(obj, "__qualname__", None), -# getattr(obj, "__code__", None) and obj.__code__.co_filename, -# getattr(obj, "__code__", None) and obj.__code__.co_firstlineno, -# ) -# origins[key] = origins.get(key, 0) + 1 -# except Exception: -# pass -# # Die 10 häufigsten Ursprünge loggen -# top = sorted(origins.items(), key=lambda x: x[1], reverse=True)[:10] -# for (mod, name, file, line), count in top: -# gc_logger.info(f"Function origin: {mod}.{name} in {file}:{line} -> {count}x") - -# def print_active_threads_and_referrers(max_referrers=3): -# """Zeigt alle aktiven Thread-Objekte und eine begrenzte Anzahl Referrers an.""" -# for obj in gc.get_objects(): -# if isinstance(obj, threading.Thread) and not obj.is_alive(): -# gc_logger.info(f"Thread: {obj} (Name: {obj.name}, Alive: {obj.is_alive()}, id={id(obj)})") -# referrers = gc.get_referrers(obj) -# gc_logger.info(f" Referrers count: {len(referrers)}") -# for ref in referrers[:max_referrers]: -# gc_logger.info(f" Referrer type: {type(ref)}, id={id(ref)}, repr={repr(ref)[:400]}") - -# Beispiel: Manuell aufrufen, wenn du eine Analyse willst -# analyze_function_origins() - def schedule_jobs(): [schedule.every().minute.at(f":{i:02d}").do(smarthome_handler).tag("algorithm") for i in range(0, 60, 5)] [schedule.every().hour.at(f":{i:02d}").do(handler.handler5Min) for i in range(0, 60, 5)] @@ -362,13 +269,10 @@ def schedule_jobs(): [schedule.every().minute.at(f":{i:02d}").do(handler.handler10Sec).tag("algorithm") for i in range(0, 60, 10)] # 30 Sekunden Handler, der die Locks überwacht, Deadlocks erkennt, loggt und ggf. den Prozess beendet schedule.every(30).seconds.do(handler.monitor_handler_locks, max_runtime=600) - # GC-Analyse alle 5 Minuten - schedule.every(5).minutes.do(write_gc_stats) try: log.debug("Start openWB2.service") - tracemalloc.start() old_memory_usage = 0 loadvars_ = loadvars.Loadvars() data.data_init(loadvars_.event_module_update_completed) From e4573051ca34f18efc587b0a86004acb0c9f6460 Mon Sep 17 00:00:00 2001 From: LKuemmel Date: Wed, 15 Oct 2025 10:47:43 +0200 Subject: [PATCH 16/16] typo --- packages/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/main.py b/packages/main.py index 176c869318..2fb8b4038b 100755 --- a/packages/main.py +++ b/packages/main.py @@ -8,6 +8,7 @@ import threading import sys import functools + # als erstes logging initialisieren, damit auch ImportError geloggt werden logger.setup_logging() log = logging.getLogger()