Add a new alert rule for reporting the MAO failure
@@ -4,11 +4,11 @@ metadata:
labels:
prometheus: k8s
role: alert-rules
- name: machineapioperator-rules
+ name: machine-api-operator-prometheus-rules
namespace: openshift-machine-api
spec:
groups:
- - name: general.rules
+ - name: machine-without-valid-node-ref
rules:
- alert: MachineWithoutValidNode
expr: |
@@ -18,25 +18,23 @@ spec:
severity: critical
annotations:
message: "machine {{ $labels.name }} does not have valid node reference"
-
----
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- labels:
- prometheus: k8s
- role: alert-rules
- name: machineapioperator-rules
- namespace: openshift-machine-api
-spec:
- groups:
- - name: general.rules
+ - name: machine-api-operator-metrics-collector-up
rules:
- - alert: MachineMAOMetricsDown
+ - alert: MachineAPIOperatorMetricsCollectionFailing
expr: |
mapi_mao_collector_up == 0
for: 3m
labels:
severity: critical
annotations:
- message: "machine api MAO metric {{ $labels.kind }} is not being reported"
+ message: "machine api operator metrics collection is failing. For more details: oc logs <machine-api-operator-pod-name> -n openshift-machine-api"
+ - name: machine-api-operator-down
+ rules:
+ - alert: MachineAPIOperatorDown
+ expr: |
+ absent(up{job="machine-api-operator"} == 1)
+ for: 3m
+ labels:
+ severity: critical
+ annotations:
+ message: "machine api operator is down"
