update prom rules for rabbit v4.1 (#1883)

Rule 'rabbitmq-tcp-sockets-near-limit' removed due to removed metrics:
* rabbitmq_process_open_tcp_sockets
* rabbitmq_process_max_tcp_sockets

Author: awoimbee

observability/prometheus/rules/rabbitmq-per-object/queue-has-no-consumers.yml

@@ -14,7 +14,7 @@ spec:
       expr: |
           (
             ((rabbitmq_detailed_queue_consumers{vhost="/", queue=~".*"} == 0) + rabbitmq_detailed_queue_messages) > 0
-          ) * on (instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info
+          ) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
       for: 10m
       annotations:
         description: |

observability/prometheus/rules/rabbitmq-per-object/queue-is-growing.yml

@@ -15,7 +15,7 @@ spec:
       expr: |
           (
               avg_over_time(rabbitmq_detailed_queue_messages[10m]) - avg_over_time(rabbitmq_detailed_queue_messages[10m] offset 1m) > 1
-          ) * on (instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info
+          ) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
       for: 10m
       annotations:
         description: |

observability/prometheus/rules/rabbitmq/cluster-alarms.yml

@@ -14,7 +14,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_memory_used_watermark[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:
@@ -30,7 +30,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_free_disk_space_watermark[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:
@@ -46,7 +46,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_file_descriptor_limit[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:

observability/prometheus/rules/rabbitmq/container-restarts.yml

@@ -12,9 +12,9 @@ spec:
     rules:
     - alert: ContainerRestarts
       expr: |
-        increase(kube_pod_container_status_restarts_total[10m]) * on(namespace, pod, container) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
-        >=
-        1
+        increase(kube_pod_container_status_restarts_total[10m])
+        * on(namespace, pod, container) group_left(rabbitmq_cluster) max by (namespace, pod, container, rabbitmq_cluster) (rabbitmq_identity_info)
+        >= 1
       for: 5m
       annotations:
         description: |

observability/prometheus/rules/rabbitmq/file-descriptors-near-limit.yml

@@ -12,9 +12,9 @@ spec:
     rules:
     - alert: FileDescriptorsNearLimit
       expr: |
-        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (max_over_time(rabbitmq_process_open_fds[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (max_over_time(rabbitmq_process_open_fds[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info))
         /
-        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (rabbitmq_process_max_fds  * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info))
         > 0.8
       for: 10m
       annotations:

observability/prometheus/rules/rabbitmq/high-connection-churn.yml

@@ -13,15 +13,27 @@ spec:
     - alert: HighConnectionChurn
       expr: |
         (
-          sum(rate(rabbitmq_connections_closed_total[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by(namespace, rabbitmq_cluster)
+          sum by (namespace, rabbitmq_cluster) (
+            rate(rabbitmq_connections_closed_total[5m])
+            * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+          )
           +
-          sum(rate(rabbitmq_connections_opened_total[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by(namespace, rabbitmq_cluster)
+          sum by (namespace, rabbitmq_cluster) (
+            rate(rabbitmq_connections_opened_total[5m])
+            * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+          )
         )
         /
-        sum (rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by (namespace, rabbitmq_cluster)
+        sum by (namespace, rabbitmq_cluster) (
+          rabbitmq_connections
+          * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+        )
         > 0.1
         unless
-        sum (rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by (namespace, rabbitmq_cluster)
+        sum by (namespace, rabbitmq_cluster) (
+          rabbitmq_connections
+          * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+        )
         < 100
       for: 10m
       annotations:

observability/prometheus/rules/rabbitmq/insufficient-established-erlang-distribution-links.yml

@@ -13,11 +13,11 @@ spec:
     - alert: InsufficientEstablishedErlangDistributionLinks
       # erlang_vm_dist_node_state: 1=pending, 2=up_pending, 3=up
       expr: |
-        count by (namespace, rabbitmq_cluster) (erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster) == 3)
+        count by (namespace, rabbitmq_cluster) (erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info) == 3)
         <
-         count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+         count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info))
          *
-        (count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) -1 )
+        (count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)) -1 )
       for: 10m
       annotations:
         description: |

observability/prometheus/rules/rabbitmq/low-disk-watermark-predicted.yml

@@ -14,13 +14,13 @@ spec:
       # The 2nd condition ensures that data points are available until 24 hours ago such that no false positive alerts are triggered for newly created RabbitMQ clusters.
       expr: |
         (
-          predict_linear(rabbitmq_disk_space_available_bytes[24h], 60*60*24) * on (instance, pod) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          predict_linear(rabbitmq_disk_space_available_bytes[24h], 60*60*24) * on (instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info)
           <
-          rabbitmq_disk_space_available_limit_bytes * on (instance, pod) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          rabbitmq_disk_space_available_limit_bytes * on (instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info)
         )
         and
         (
-          count_over_time(rabbitmq_disk_space_available_limit_bytes[2h] offset 22h) * on (instance, pod) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          count_over_time(rabbitmq_disk_space_available_limit_bytes[2h] offset 22h) * on (instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info)
           >
           0
         )

observability/prometheus/rules/rabbitmq/tcp-sockets-near-limit.yml

@@ -12,10 +12,10 @@ spec:
     rules:
     - alert: UnroutableMessages
       expr: |
-        sum by(namespace, rabbitmq_cluster) (increase(rabbitmq_channel_messages_unroutable_dropped_total[5m]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info)
+        sum by(namespace, rabbitmq_cluster) (increase(rabbitmq_channel_messages_unroutable_dropped_total[5m]) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info))
         >= 1
         or
-        sum by(namespace, rabbitmq_cluster) (increase(rabbitmq_channel_messages_unroutable_returned_total[5m]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info)
+        sum by(namespace, rabbitmq_cluster) (increase(rabbitmq_channel_messages_unroutable_returned_total[5m]) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info))
         >= 1
       annotations:
         description: |