Merge pull request #1173 from stackhpc/ping-alerts

Make packet drop alert configurable

Author: Alex-Welsh

etc/kayobe/kolla/config/prometheus/ceph.rules

@@ -137,25 +137,25 @@ groups:
     annotations:
       description: "Root volume (OSD and MON store) is dangerously full: {{ $value | humanize }}% free."
 
-  # alert on nic packet errors and drops rates > 1 packet/s
+  # alert on nic packet errors and drops rates > alertmanager_packet_drop_threshold packet/s
   - alert: NetworkPacketsDropped
-    expr: irate(node_network_receive_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) + irate(node_network_transmit_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) > 1
+    expr: irate(node_network_receive_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) + irate(node_network_transmit_drop_total{device!~"lo|br.*|.*-ovs|tap.*"}[5m]) > {% endraw %}{{ alertmanager_packet_drop_threshold }}{% raw %}
     labels:
       severity: warning
     annotations:
       description: >
-        Node {{ $labels.instance }} experiences packet drop > 1
+        Node {{ $labels.instance }} experiences packet drop > {% endraw %}{{ alertmanager_packet_drop_threshold }}{% raw %}
         packet/s on interface {{ $labels.device }}.
 
   - alert: NetworkPacketErrors
     expr: |
       irate(node_network_receive_errs_total{device!="lo"}[5m]) +
-      irate(node_network_transmit_errs_total{device!="lo"}[5m]) > 1
+      irate(node_network_transmit_errs_total{device!="lo"}[5m]) > {% endraw %}{{ alertmanager_packet_errors_threshold }}{% raw %}
     labels:
       severity: warning
     annotations:
       description: >
-        Node {{ $labels.instance }} experiences packet errors > 1
+        Node {{ $labels.instance }} experiences packet errors > {% endraw %}{{ alertmanager_packet_errors_threshold }}{% raw %}
         packet/s on interface {{ $labels.device }}.
 
   - alert: StorageFillingUp

etc/kayobe/stackhpc-monitoring.yml

@@ -18,6 +18,14 @@ alertmanager_warn_network_bond_single_link: true
 alertmanager_node_free_swap_warning_threshold_ratio: 0.25
 alertmanager_node_free_swap_critical_threshold_ratio: 0.1
 
+# Threshold to trigger an alert for dropped packets, measured in packets/s
+# averaged over 5 minutes.
+alertmanager_packet_drop_threshold: 1
+
+# Threshold to trigger an alert for packet receive/transmit errors, measured in
+# packets/s averaged over 5 minutes.
+alertmanager_packet_errors_threshold: 1
+
 ###############################################################################
 # Exporter configuration