Improve UAPI error handler

Replace the fixed count-based error handler
with a duration-based mechanism and add
FeatureHealthMonitoring feature for configuring it.

Signed-off-by: Lukas Pukenis <lukas.pukenis@nordsec.com>

Author: LukasPukenis

.unreleased/LLT-6859

@@ -0,0 +1 @@
+Fix critical error handler and allow it to be configurable

crates/telio-model/src/event.rs

@@ -22,7 +22,7 @@ macro_rules! report_event {
 }
 
 /// Error levels. Used for app to decide what to do with `telio` device when error happens.
-#[derive(Clone, Debug, Default, Serialize)]
+#[derive(Clone, Debug, Default, Serialize, Eq, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum ErrorLevel {
     /// The error level is critical (highest priority)
@@ -37,7 +37,7 @@ pub enum ErrorLevel {
 }
 
 /// Error code. Common error code representation (for statistics).
-#[derive(Clone, Debug, Default, Serialize)]
+#[derive(Clone, Debug, Default, Serialize, Eq, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum ErrorCode {
     /// There is no error in the execution

crates/telio-model/src/features.rs

@@ -93,15 +93,19 @@ pub struct FeatureBatching {
 }
 
 /// Configurable features for Wireguard peers
-#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, SmartDefault, PartialEq, Eq, Serialize, Deserialize)]
 #[cfg_attr(test, derive(proptest_derive::Arbitrary))]
+#[serde(default)]
 pub struct FeatureWireguard {
     /// Configurable persistent keepalive periods for wireguard peers
     #[serde(default)]
     pub persistent_keepalive: FeaturePersistentKeepalive,
     /// Configurable wireguard polling period
     #[serde(default)]
     pub polling: FeaturePolling,
+    /// Configurable Wireguard health monitor
+    #[serde(default)]
+    pub health_monitoring: FeatureHealthMonitoring,
     /// Configurable up/down behavior of WireGuard-NT adapter. See RFC LLT-0089 for details
     #[serde(default)]
     pub enable_dynamic_wg_nt_control: bool,
@@ -148,6 +152,19 @@ pub struct FeaturePersistentKeepalive {
     pub stun: Option<u32>,
 }
 
+/// Configurable Wireguard health monitor
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize, SmartDefault)]
+#[serde(default)]
+#[cfg_attr(test, derive(proptest_derive::Arbitrary))]
+pub struct FeatureHealthMonitoring {
+    /// Minimum interval between two UAPI errors required before escalating to a critical error (in seconds) [default 10s]
+    #[default(10)]
+    pub failure_escalation_min_interval_secs: u32,
+    /// Maximum acceptable gap between consecutive UAPI polls before suspecting system suspension (in seconds) [default 60s]
+    #[default(60)]
+    pub expected_max_uapi_poll_interval: u32,
+}
+
 /// Configurable Wireguard polling period
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, SmartDefault)]
 #[serde(default)]
@@ -698,6 +715,10 @@ mod tests {
                     "wireguard_polling_period": 1000,
                     "wireguard_polling_period_after_state_change": 50
                 },
+                "health_monitoring": {
+                    "failure_escalation_min_interval_secs": 44,
+                    "expected_max_uapi_poll_interval": 55
+                },
                 "enable_dynamic_wg_nt_control": true,
                 "skt_buffer_size": 123456,
                 "inter_thread_channel_size": 123456,
@@ -801,6 +822,10 @@ mod tests {
                             wireguard_polling_period: 1000,
                             wireguard_polling_period_after_state_change: 50
                         },
+                        health_monitoring: FeatureHealthMonitoring {
+                            failure_escalation_min_interval_secs: 44,
+                            expected_max_uapi_poll_interval: 55
+                        },
                         enable_dynamic_wg_nt_control: true,
                         skt_buffer_size: Some(123456),
                         inter_thread_channel_size: Some(123456),