Add CPU and memory alerts for the master nodes
@@ -57,3 +57,24 @@ spec:
severity: warning
annotations:
message: "System memory usage of {{ $value | humanize }} on {{ $labels.node }} exceeds 90% of the reservation. Reserved memory ensures system processes can function even when the node is fully allocated and protects against workload out of memory events impacting the proper functioning of the node. The default reservation is expected to be sufficient for most configurations and should be increased (https://docs.openshift.com/container-platform/latest/nodes/nodes/nodes-nodes-managing.html) when running nodes with high numbers of pods (either due to rate of change or at steady state)."
+ - name: master-nodes-high-memory-usage
+ rules:
+ - alert: MasterNodesHighMemoryUsage
+ expr: |
+ ((sum(node_memory_MemTotal_bytes AND on (instance) label_replace( kube_node_role{role="master"}, "instance", "$1", "node", "(.+)" )) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes AND on (instance) label_replace( kube_node_role{role="master"}, "instance", "$1", "node", "(.+)" ))) / sum(node_memory_MemTotal_bytes AND on (instance) label_replace( kube_node_role{role="master"}, "instance", "$1", "node", "(.+)" )) * 100) > 90
+ for: 15m
+ labels:
+ severity: warning
+ annotations:
+ message: "Memory usage of {{ $value | humanize }} on {{ $labels.node }} exceeds by 90%. Master nodes starved of memory could result in degraded performance of the control plane."
+ - name: master-nodes-high-cpu-usage
+ rules:
+ - alert: MasterNodesHighCPUUsage
+ expr: |
+ sum(node_load1 AND on (instance) label_replace( kube_node_role{role="master"}, "instance", "$1", "node", "(.+)" )) > (count(node_load1 AND on (instance) label_replace( kube_node_role{role="master"}, "instance", "$1", "node", "(.+)" )) * 3)
+ for: 15m
+ labels:
+ severity: warning
+ annotations:
+ message: "Cumulative load of {{ $value | humanize }} on {{ $labels.node }} is very high. Master nodes starved of cpu could result in degraded performance of the control pane."
+
