apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-kubernets-monitoring-alerts
  namespace: infra
data:
  # THIS FILE WAS GENERATED FROM https://github.com/kubernetes-monitoring/kubernetes-mixin
  prometheus_alerts.yml: |
    "groups":
      - "name": "kubernetes-absent"
        "rules":
        - "alert": "KubeAPIDown"
          "annotations":
            "message": "KubeAPI has disappeared from Prometheus target discovery."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown"
          "expr": |
            absent(up{job="kube-apiserver"} == 1)
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeControllerManagerDown"
          "annotations":
            "message": "KubeControllerManager has disappeared from Prometheus target discovery."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown"
          "expr": |
            absent(up{job="kube-controller-manager"} == 1)
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeSchedulerDown"
          "annotations":
            "message": "KubeScheduler has disappeared from Prometheus target discovery."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown"
          "expr": |
            absent(up{job="kube-scheduler"} == 1)
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeletDown"
          "annotations":
            "message": "Kubelet has disappeared from Prometheus target discovery."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown"
          "expr": |
            absent(up{job="kubelet"} == 1)
          "for": "15m"
          "labels":
            "severity": "critical"
      - "name": "kubernetes-apps"
        "rules":
        - "alert": "KubePodCrashLooping"
          "annotations":
            "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 5 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping"
          "expr": |
            rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubePodNotReady"
          "annotations":
            "message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready"
          "expr": |
            sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Failed|Pending|Unknown"} * on(namespace, pod) group_left(owner_kind) kube_pod_owner{owner_kind!="Job"}) > 0
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeDeploymentGenerationMismatch"
          "annotations":
            "message": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch"
          "expr": |
            kube_deployment_status_observed_generation{job="kube-state-metrics"}
              !=
            kube_deployment_metadata_generation{job="kube-state-metrics"}
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeDeploymentReplicasMismatch"
          "annotations":
            "message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch"
          "expr": |
            kube_deployment_spec_replicas{job="kube-state-metrics"}
              !=
            kube_deployment_status_replicas_available{job="kube-state-metrics"}
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeStatefulSetReplicasMismatch"
          "annotations":
            "message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch"
          "expr": |
            kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
              !=
            kube_statefulset_status_replicas{job="kube-state-metrics"}
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeStatefulSetGenerationMismatch"
          "annotations":
            "message": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch"
          "expr": |
            kube_statefulset_status_observed_generation{job="kube-state-metrics"}
              !=
            kube_statefulset_metadata_generation{job="kube-state-metrics"}
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeStatefulSetUpdateNotRolledOut"
          "annotations":
            "message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout"
          "expr": |
            max without (revision) (
              kube_statefulset_status_current_revision{job="kube-state-metrics"}
                unless
              kube_statefulset_status_update_revision{job="kube-state-metrics"}
            )
              *
            (
              kube_statefulset_replicas{job="kube-state-metrics"}
                !=
              kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
            )
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeDaemonSetRolloutStuck"
          "annotations":
            "message": "Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck"
          "expr": |
            kube_daemonset_status_number_ready{job="kube-state-metrics"}
              /
            kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} < 1.00
          "for": "15m"
          "labels":
            "severity": "critical"
        - "alert": "KubeDaemonSetNotScheduled"
          "annotations":
            "message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled"
          "expr": |
            kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
              -
            kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
          "for": "10m"
          "labels":
            "severity": "warning"
        - "alert": "KubeDaemonSetMisScheduled"
          "annotations":
            "message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled"
          "expr": |
            kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
          "for": "10m"
          "labels":
            "severity": "warning"
        - "alert": "KubeCronJobRunning"
          "annotations":
            "message": "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning"
          "expr": |
            time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600
          "for": "1h"
          "labels":
            "severity": "warning"
        - "alert": "KubeJobCompletion"
          "annotations":
            "message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than one hour to complete."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion"
          "expr": |
            kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"}  > 0
          "for": "1h"
          "labels":
            "severity": "warning"
        - "alert": "KubeJobFailed"
          "annotations":
            "message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed"
          "expr": |
            kube_job_failed{job="kube-state-metrics"}  > 0
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeHpaReplicasMismatch"
          "annotations":
            "message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch"
          "expr": |
            (kube_hpa_status_desired_replicas{job="kube-state-metrics"}
              !=
            kube_hpa_status_current_replicas{job="kube-state-metrics"})
              and
            changes(kube_hpa_status_current_replicas[15m]) == 0
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeHpaMaxedOut"
          "annotations":
            "message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout"
          "expr": |
            kube_hpa_status_current_replicas{job="kube-state-metrics"}
              ==
            kube_hpa_spec_max_replicas{job="kube-state-metrics"}
          "for": "15m"
          "labels":
            "severity": "warning"
      - "name": "kubernetes-resources"
        "rules":
        - "alert": "KubeCPUOvercommit"
          "annotations":
            "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
          "expr": |
            sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum)
              /
            sum(kube_node_status_allocatable_cpu_cores)
              >
            (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)
          "for": "5m"
          "labels":
            "severity": "warning"
        - "alert": "KubeMemOvercommit"
          "annotations":
            "message": "Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit"
          "expr": |
            sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum)
              /
            sum(kube_node_status_allocatable_memory_bytes)
              >
            (count(kube_node_status_allocatable_memory_bytes)-1)
              /
            count(kube_node_status_allocatable_memory_bytes)
          "for": "5m"
          "labels":
            "severity": "warning"
        - "alert": "KubeCPUOvercommit"
          "annotations":
            "message": "Cluster has overcommitted CPU resource requests for Namespaces."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
          "expr": |
            sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
              /
            sum(kube_node_status_allocatable_cpu_cores)
              > 1.5
          "for": "5m"
          "labels":
            "severity": "warning"
        - "alert": "KubeMemOvercommit"
          "annotations":
            "message": "Cluster has overcommitted memory resource requests for Namespaces."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit"
          "expr": |
            sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
              /
            sum(kube_node_status_allocatable_memory_bytes{job="node-exporter"})
              > 1.5
          "for": "5m"
          "labels":
            "severity": "warning"
        - "alert": "KubeQuotaExceeded"
          "annotations":
            "message": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded"
          "expr": |
            kube_resourcequota{job="kube-state-metrics", type="used"}
              / ignoring(instance, job, type)
            (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
              > 0.90
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "CPUThrottlingHigh"
          "annotations":
            "message": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh"
          "expr": |
            sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
              /
            sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
              > ( 25 / 100 )
          "for": "15m"
          "labels":
            "severity": "warning"
      - "name": "kubernetes-storage"
        "rules":
        - "alert": "KubePersistentVolumeUsageCritical"
          "annotations":
            "message": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical"
          "expr": |
            kubelet_volume_stats_available_bytes{job="kubelet"}
              /
            kubelet_volume_stats_capacity_bytes{job="kubelet"}
              < 0.03
          "for": "1m"
          "labels":
            "severity": "critical"
        - "alert": "KubePersistentVolumeFullInFourDays"
          "annotations":
            "message": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays"
          "expr": |
            (
              kubelet_volume_stats_available_bytes{job="kubelet"}
                /
              kubelet_volume_stats_capacity_bytes{job="kubelet"}
            ) < 0.15
            and
            predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
          "for": "5m"
          "labels":
            "severity": "critical"
        - "alert": "KubePersistentVolumeErrors"
          "annotations":
            "message": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors"
          "expr": |
            kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
          "for": "5m"
          "labels":
            "severity": "critical"
      - "name": "kubernetes-system"
        "rules":
        - "alert": "KubeNodeNotReady"
          "annotations":
            "message": "{{ $labels.node }} has been unready for more than 15 minutes."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready"
          "expr": |
            kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeVersionMismatch"
          "annotations":
            "message": "There are {{ $value }} different semantic versions of Kubernetes components running."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch"
          "expr": |
            count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeClientErrors"
          "annotations":
            "message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'"
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors"
          "expr": |
            (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
              /
            sum(rate(rest_client_requests_total[5m])) by (instance, job))
            > 0.01
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeletTooManyPods"
          "annotations":
            "message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods"
          "expr": |
            max(max(kubelet_running_pod_count{job="kubelet"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"}) by(node) > 0.95
          "for": "15m"
          "labels":
            "severity": "warning"
        - "alert": "KubeAPILatencyHigh"
          "annotations":
            "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh"
          "expr": |
            cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="kube-apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1
          "for": "10m"
          "labels":
            "severity": "warning"
        - "alert": "KubeAPILatencyHigh"
          "annotations":
            "message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh"
          "expr": |
            cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="kube-apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4
          "for": "10m"
          "labels":
            "severity": "critical"
        - "alert": "KubeAPIErrorsHigh"
          "annotations":
            "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
          "expr": |
            sum(rate(apiserver_request_total{job="kube-apiserver",code=~"^(?:5..)$"}[5m]))
              /
            sum(rate(apiserver_request_total{job="kube-apiserver"}[5m])) > 0.03
          "for": "10m"
          "labels":
            "severity": "critical"
        - "alert": "KubeAPIErrorsHigh"
          "annotations":
            "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
          "expr": |
            sum(rate(apiserver_request_total{job="kube-apiserver",code=~"^(?:5..)$"}[5m]))
              /
            sum(rate(apiserver_request_total{job="kube-apiserver"}[5m])) > 0.01
          "for": "10m"
          "labels":
            "severity": "warning"
        - "alert": "KubeAPIErrorsHigh"
          "annotations":
            "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
          "expr": |
            sum(rate(apiserver_request_total{job="kube-apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb)
              /
            sum(rate(apiserver_request_total{job="kube-apiserver"}[5m])) by (resource,subresource,verb) > 0.10
          "for": "10m"
          "labels":
            "severity": "critical"
        - "alert": "KubeAPIErrorsHigh"
          "annotations":
            "message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
          "expr": |
            sum(rate(apiserver_request_total{job="kube-apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb)
              /
            sum(rate(apiserver_request_total{job="kube-apiserver"}[5m])) by (resource,subresource,verb) > 0.05
          "for": "10m"
          "labels":
            "severity": "warning"
        - "alert": "KubeClientCertificateExpiration"
          "annotations":
            "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
          "expr": |
            apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800
          "labels":
            "severity": "warning"
        - "alert": "KubeClientCertificateExpiration"
          "annotations":
            "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours."
            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
          "expr": |
            apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400
          "labels":
            "severity": "critical"
      