add prometheus rules to track virt-api REST health
@@ -60,7 +60,7 @@ func NewServiceMonitorCR(namespace string, monitorNamespace string, insecureSkip
func NewPrometheusRuleCR(namespace string, workloadUpdatesEnabled bool) *v1.PrometheusRule {
return &v1.PrometheusRule{
TypeMeta: v12.TypeMeta{
- APIVersion: v1.SchemeGroupVersion.String(),
+ APIVersion: v12.SchemeGroupVersion.String(),
Kind: "PrometheusRule",
},
ObjectMeta: v12.ObjectMeta{
@@ -434,6 +434,52 @@ func NewPrometheusRuleSpec(ns string, workloadUpdatesEnabled bool) *v1.Prometheu
"severity": "critical",
},
},
+ {
+ Record: "vec_by_virt_apis_all_client_rest_requests_in_last_5m",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod=~'virt-api-.*', namespace='%s'}[5m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_apis_all_client_rest_requests_in_last_hour",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod=~'virt-api-.*', namespace='%s'}[60m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_apis_failed_client_rest_requests_in_last_5m",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod=~'virt-api-.*', namespace='%s', code=~'(4|5)[0-9][0-9]'}[5m]))", ns),
+ ),
+ },
+ {
+ Record: "vec_by_virt_apis_failed_client_rest_requests_in_last_hour",
+ Expr: intstr.FromString(
+ fmt.Sprintf("sum by (pod) (sum_over_time(rest_client_requests_total{pod=~'virt-api-.*', namespace='%s', code=~'(4|5)[0-9][0-9]'}[60m]))", ns),
+ ),
+ },
+ {
+ Alert: "VirtApiRESTErrorsHigh",
+ Expr: intstr.FromString("(vec_by_virt_apis_failed_client_rest_requests_in_last_hour / vec_by_virt_apis_all_client_rest_requests_in_last_hour) >= 0.05"),
+ For: "5m",
+ Annotations: map[string]string{
+ "summary": "More than 5% of the rest calls failed in virt-api for the last hour",
+ },
+ Labels: map[string]string{
+ "severity": "warning",
+ },
+ },
+ {
+ Alert: "VirtApiRESTErrorsBurst",
+ Expr: intstr.FromString("(vec_by_virt_apis_failed_client_rest_requests_in_last_5m / vec_by_virt_apis_all_client_rest_requests_in_last_5m) >= 0.8"),
+ For: "5m",
+ Annotations: map[string]string{
+ "summary": "More than 80% of the rest calls failed in virt-api for the last 5 minutes",
+ },
+ Labels: map[string]string{
+ "severity": "critical",
+ },
+ },
{
Record: "kubevirt_vm_container_free_memory_bytes",
Expr: intstr.FromString("sum by(pod, container) ( kube_pod_container_resource_limits_memory_bytes{pod=~'virt-launcher-.*', container='compute'} - on(pod,container) container_memory_working_set_bytes{pod=~'virt-launcher-.*', container='compute'})"),
