Add basic alert rules for virt-operator
This PR features only alert rules that can be computed out of metrics
that we already have in the cluster and does not require any changes to
virt-operator itself.
@@ -415,6 +415,68 @@ func NewPrometheusRuleSpec(ns string) *promv1.PrometheusRuleSpec {
"summary": "More than 80% of the rest calls failed in virt-controller for the last 5 minutes",
},
},
+ {
+ Record: "num_of_running_virt_operators",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum(up{namespace='%s', pod='virt-operator-.*'})", ns),
+ ),
+ },
+ {
+ Alert: "VirtOperatorDown",
+ Expr: intstr.FromString("num_of_running_virt_operators == 0"),
+ For: "5m",
+ Annotations: map[string]string{
+ "summary": "All virt-operator servers are down.",
+ },
+ },
+ {
+ Alert: "LowVirtOperatorCount",
+ Expr: intstr.FromString("(num_of_allocatable_nodes > 1) and (num_of_running_virt_operators < 2)"),
+ For: "60m",
+ Annotations: map[string]string{
+ "summary": "More than one virt-operator should be running if more than one worker nodes exist.",
+ },
+ },
+ {
+ Record: "vec_by_virt_operators_all_client_rest_requests_in_last_hour",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod='virt-operator-.*', namespace='%s', code=~'[2][0-9][0-9]'}[60m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_operators_failed_client_rest_requests_in_last_hour",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod='virt-operator-.*', namespace='%s', code=~'[^2][0-9][0-9]'}[60m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_operators_all_client_rest_requests_in_last_5m",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod='virt-operator-.*', namespace='%s', code=~'[2][0-9][0-9]'}[5m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_operators_failed_client_rest_requests_in_last_5m",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod='virt-operator-.*', namespace='%s', code=~'[^2][0-9][0-9]'}[5m]))", ns),
+ ),
+ },
+ {
+ Alert: "VirtOperatorRESTErrorsHigh",
+ Expr: intstr.FromString("(vec_by_virt_operators_failed_client_rest_requests_in_last_hour / vec_by_virt_operators_all_client_rest_requests_in_last_hour) >= 0.05"),
+ For: "5m",
+ Annotations: map[string]string{
+ "summary": "More than 5% of the rest calls failed in virt-operator for the last hour",
+ },
+ },
+ {
+ Alert: "VirtOperatorRESTErrorsBurst",
+ Expr: intstr.FromString("(vec_by_virt_operators_failed_client_rest_requests_in_last_5m / vec_by_virt_operators_all_client_rest_requests_in_last_5m) >= 0.8"),
+ For: "5m",
+ Annotations: map[string]string{
+ "summary": "More than 80% of the rest calls failed in virt-operator for the last 5 minutes",
+ },
+ },
},
},
},
