Merge pull request #1044 from stackhpc/bond-single-link-alert

Add alert to detect bonds with a single link

Author: priteau

etc/kayobe/kolla/config/prometheus/system.rules

@@ -104,7 +104,22 @@ groups:
     annotations:
       summary: Host network bond degraded (instance {{ $labels.instance }})
       description: "Bond {{ $labels.master }} degraded on {{ $labels.instance }}"
+{% endraw %}
 
+{% if alertmanager_warn_network_bond_single_link | bool %}
+{% raw %}
+  - alert: HostNetworkBondSingleLink
+    expr: node_bonding_slaves == 1
+    for: 2m
+    labels:
+      severity: warning
+    annotations:
+      summary: Host network bond with a single link (instance {{ $labels.instance }})
+      description: "Bond {{ $labels.master }} configured with a single link on {{ $labels.instance }}"
+{% endraw %}
+{% endif %}
+
+{% raw %}
   - alert: HostConntrackLimit
     expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
     for: 5m

etc/kayobe/stackhpc-monitoring.yml

@@ -8,6 +8,10 @@
 # of free memory is lower than this value an alert will be triggered.
 alertmanager_low_memory_threshold_gib: 5
 
+# Whether to raise an alert if any network bond is configured with a single
+# link. Change to false to disable this alert.
+alertmanager_warn_network_bond_single_link: true
+
 ###############################################################################
 # Exporter configuration
 

releasenotes/notes/network-bond-single-link-766adf41a3c2fd4e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds a new Prometheus alert ``HostNetworkBondSingleLink`` which will be
+    raised when a bond is configured with only one member. This can happen when
+    NetworkManager detects that a bond member is down at boot time. This alert
+    can be disabled by setting ``alertmanager_warn_network_bond_single_link``
+    to ``false``.