---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kubernetes-monitoring-alertingrules
  namespace: open-cluster-management-addon-observability
data:
  kubernetes-monitoring-alertingrules.yaml: |
    groups:
    - name: kube-state-metrics
      rules:
      - alert: KubeStateMetricsListErrors
        annotations:
          description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
          summary: kube-state-metrics is experiencing errors in list operations.
        expr: |
          (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
      - alert: KubeStateMetricsWatchErrors
        annotations:
          description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
          summary: kube-state-metrics is experiencing errors in watch operations.
        expr: |
          (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
    - name: kubernetes-apps
      rules:
      - alert: KubePodCrashLooping
        annotations:
          description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 10 minutes.
          summary: Pod is crash looping.
        expr: |
          rate(kube_pod_container_status_restarts_total{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}[10m]) * 60 * 5 > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubePodNotReady
        annotations:
          description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
          summary: Pod has been in a non-ready state for more than 15 minutes.
        expr: |
          sum by (namespace, pod) (
            max by(namespace, pod) (
              kube_pod_status_phase{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", phase=~"Pending|Unknown"}
            ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (
              1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})
            )
          ) > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeDeploymentGenerationMismatch
        annotations:
          description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
          summary: Deployment generation mismatch due to possible roll-back
        expr: |
          kube_deployment_status_observed_generation{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            !=
          kube_deployment_metadata_generation{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetReplicasMismatch
        annotations:
          description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
          summary: Deployment has not matched the expected number of replicas.
        expr: |
          (
            kube_statefulset_status_replicas_ready{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
              !=
            kube_statefulset_status_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
          ) and (
            changes(kube_statefulset_status_replicas_updated{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}[10m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetGenerationMismatch
        annotations:
          description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
          summary: StatefulSet generation mismatch due to possible roll-back
        expr: |
          kube_statefulset_status_observed_generation{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            !=
          kube_statefulset_metadata_generation{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
      - alert: KubeStatefulSetUpdateNotRolledOut
        annotations:
          description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
          summary: StatefulSet update has not been rolled out.
        expr: |
          (
            max without (revision) (
              kube_statefulset_status_current_revision{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                unless
              kube_statefulset_status_update_revision{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            )
              *
            (
              kube_statefulset_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                !=
              kube_statefulset_status_replicas_updated{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            )
          )  and (
            changes(kube_statefulset_status_replicas_updated{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}[5m])
              ==
            0
          )
        for: 15m
        labels:
          severity: warning
      - alert: KubeDaemonSetRolloutStuck
        annotations:
          description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 30 minutes.
          summary: DaemonSet rollout is stuck.
        expr: |
          (
            (
              kube_daemonset_status_current_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            ) or (
              kube_daemonset_status_number_misscheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                !=
              0
            ) or (
              kube_daemonset_updated_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            ) or (
              kube_daemonset_status_number_available{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
                !=
              kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            )
          ) and (
            changes(kube_daemonset_updated_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}[5m])
              ==
            0
          )
        for: 30m
        labels:
          severity: warning
      - alert: KubeContainerWaiting
        annotations:
          description: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.
          summary: Pod container waiting longer than 1 hour
        expr: |
          sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}) > 0
        for: 1h
        labels:
          severity: warning
      - alert: KubeDaemonSetNotScheduled
        annotations:
          description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
          summary: DaemonSet pods are not scheduled.
        expr: |
          kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            -
          kube_daemonset_status_current_number_scheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"} > 0
        for: 10m
        labels:
          severity: warning
      - alert: KubeDaemonSetMisScheduled
        annotations:
          description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
          summary: DaemonSet pods are misscheduled.
        expr: |
          kube_daemonset_status_number_misscheduled{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"} > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeJobCompletion
        annotations:
          description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete.
          summary: Job did not complete in time
        expr: |
          kube_job_spec_completions{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"} - kube_job_status_succeeded{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}  > 0
        for: 12h
        labels:
          severity: warning
      - alert: KubeJobFailed
        annotations:
          description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
          summary: Job failed to complete.
        expr: |
          kube_job_failed{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}  > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeHpaReplicasMismatch
        annotations:
          description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.
          summary: HPA has not matched descired number of replicas.
        expr: |
          (kube_hpa_status_desired_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            !=
          kube_hpa_status_current_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"})
            and
          (kube_hpa_status_current_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            >
          kube_hpa_spec_min_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"})
            and
          (kube_hpa_status_current_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            <
          kube_hpa_spec_max_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"})
            and
          changes(kube_hpa_status_current_replicas[15m]) == 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeHpaMaxedOut
        annotations:
          description: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes.
          summary: HPA is running at max replicas
        expr: |
          kube_hpa_status_current_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
            ==
          kube_hpa_spec_max_replicas{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"}
        for: 15m
        labels:
          severity: warning
    - name: kubernetes-resources
      rules:
      - alert: KubeCPUOvercommit
        annotations:
          description: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.
          summary: Cluster has overcommitted CPU resource requests.
        expr: |
          sum(namespace_cpu:kube_pod_container_resource_requests:sum{})
            /
          sum(kube_node_status_allocatable{resource="cpu"})
            >
          (count(kube_node_status_allocatable{resource="cpu"}) -1) / count(kube_node_status_allocatable{resource="cpu"})
        for: 5m
        labels:
          severity: warning
      - alert: KubeMemoryOvercommit
        annotations:
          description: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.
          summary: Cluster has overcommitted memory resource requests.
        expr: |
          sum(namespace_memory:kube_pod_container_resource_requests_bytes:sum{})
            /
          sum(kube_node_status_allocatable{resource="memory"})
            >
          (count(kube_node_status_allocatable{resource="memory"})-1)
            /
          count(kube_node_status_allocatable{resource="memory"})
        for: 5m
        labels:
          severity: warning
      - alert: KubeCPUQuotaOvercommit
        annotations:
          description: Cluster has overcommitted CPU resource requests for Namespaces.
          summary: Cluster has overcommitted CPU resource requests.
        expr: "sum(kube_resourcequota{namespace=~\"(kube-.*|default|logging)\",job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n  /\nsum(kube_node_status_allocatable{resource=\"cpu\"}) \n  > 1.5\n"
        for: 5m
        labels:
          severity: warning
      - alert: KubeMemoryQuotaOvercommit
        annotations:
          description: Cluster has overcommitted memory resource requests for Namespaces.
          summary: Cluster has overcommitted memory resource requests.
        expr: |
          sum(kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="hard", resource="memory"})
            /
          sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"})
            > 1.5
        for: 5m
        labels:
          severity: warning
      - alert: KubeQuotaAlmostFull
        annotations:
          description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
          summary: Namespace quota is going to be full.
        expr: |
          kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="hard"} > 0)
            > 0.9 < 1
        for: 15m
        labels:
          severity: info
      - alert: KubeQuotaFullyUsed
        annotations:
          description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
          summary: Namespace quota is fully used.
        expr: |
          kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="hard"} > 0)
            == 1
        for: 15m
        labels:
          severity: info
      - alert: KubeQuotaExceeded
        annotations:
          description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
          summary: Namespace quota has exceeded the limits.
        expr: |
          kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="used"}
            / ignoring(instance, job, type)
          (kube_resourcequota{namespace=~"(kube-.*|default|logging)",job="kube-state-metrics", type="hard"} > 0)
            > 1
        for: 15m
        labels:
          severity: warning
    - name: kubernetes-storage
      rules:
      # - alert: KubePersistentVolumeFillingUp
      #   annotations:
      #     description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
      #     summary: PersistentVolume is filling up.
      #   expr: |
      #     kubelet_volume_stats_available_bytes{namespace=~"(kube-.*|default|logging)",job="kubelet", metrics_path="/metrics"}
      #       /
      #     kubelet_volume_stats_capacity_bytes{namespace=~"(kube-.*|default|logging)",job="kubelet", metrics_path="/metrics"}
      #       < 0.03
      #   for: 1m
      #   labels:
      #     severity: critical
      # - alert: KubePersistentVolumeFillingUp
      #   annotations:
      #     description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
      #     summary: PersistentVolume is filling up.
      #   expr: |
      #     (
      #       kubelet_volume_stats_available_bytes{namespace=~"(kube-.*|default|logging)",job="kubelet", metrics_path="/metrics"}
      #         /
      #       kubelet_volume_stats_capacity_bytes{namespace=~"(kube-.*|default|logging)",job="kubelet", metrics_path="/metrics"}
      #     ) < 0.15
      #     and
      #     predict_linear(kubelet_volume_stats_available_bytes{namespace=~"(kube-.*|default|logging)",job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
      #   for: 1h
      #   labels:
      #     severity: warning
      - alert: KubePersistentVolumeErrors
        annotations:
          description: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
          summary: PersistentVolume is having issues with provisioning.
        expr: |
          kube_persistentvolume_status_phase{phase=~"Failed|Pending",namespace=~"(kube-.*|default|logging)",job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: critical
    - name: kubernetes-system
      rules:
      - alert: KubeClientErrors
        annotations:
          description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
          summary: Kubernetes API server client is experiencing errors.
        expr: |
          (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
            /
          sum(rate(rest_client_requests_total[5m])) by (instance, job))
          > 0.01
        for: 15m
        labels:
          severity: warning
    - name: kube-apiserver-slos
      rules:
      - alert: KubeAPIErrorBudgetBurn
        annotations:
          description: The API server is burning too much error budget.
          summary: The API server is burning too much error budget.
        expr: |
          sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
          and
          sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
        for: 2m
        labels:
          long: 1h
          severity: critical
          short: 5m
      - alert: KubeAPIErrorBudgetBurn
        annotations:
          description: The API server is burning too much error budget.
          summary: The API server is burning too much error budget.
        expr: |
          sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
          and
          sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
        for: 15m
        labels:
          long: 6h
          severity: critical
          short: 30m
      - alert: KubeAPIErrorBudgetBurn
        annotations:
          description: The API server is burning too much error budget.
          summary: The API server is burning too much error budget.
        expr: |
          sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
          and
          sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
        for: 1h
        labels:
          long: 1d
          severity: warning
          short: 2h
      - alert: KubeAPIErrorBudgetBurn
        annotations:
          description: The API server is burning too much error budget.
          summary: The API server is burning too much error budget.
        expr: |
          sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
          and
          sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
        for: 3h
        labels:
          long: 3d
          severity: warning
          short: 6h
    - name: kubernetes-system-apiserver
      rules:
      - alert: AggregatedAPIErrors
        annotations:
          description: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.
          summary: An aggregated API has reported errors.
        expr: |
          sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[10m])) > 4
        labels:
          severity: warning
      - alert: AggregatedAPIDown
        annotations:
          description: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
          summary: An aggregated API is down.
        expr: |
          (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
        for: 15m
        labels:
          severity: warning
      - alert: KubeAPIDown
        annotations:
          description: KubeAPI has disappeared from Prometheus target discovery.
          summary: Target disappeared from Prometheus target discovery.
        expr: |
          absent(up{job="apiserver"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: KubeAPITerminatedRequests
        annotations:
          description: The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
          summary: The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
        expr: |
          sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))  / (  sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
        for: 5m
        labels:
          severity: warning
    - name: kubernetes-system-kubelet
      rules:
      - alert: KubeNodeNotReady
        annotations:
          description: '{{ $labels.node }} has been unready for more than 15 minutes.'
          summary: Node is not ready.
        expr: |
          kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeNodeUnreachable
        annotations:
          description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
          summary: Node is unreachable.
        expr: |
          (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
        for: 15m
        labels:
          severity: warning
      - alert: KubeletTooManyPods
        annotations:
          description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
          summary: Kubelet is running at capacity.
        expr: |
          count by(node) (
            (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
          )
          /
          max by(node) (
            kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
          ) > 0.95
        for: 15m
        labels:
          severity: warning
      - alert: KubeNodeReadinessFlapping
        annotations:
          description: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
          summary: Node readiness status is flapping.
        expr: |
          sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
        for: 15m
        labels:
          severity: warning
      - alert: KubeletPlegDurationHigh
        annotations:
          description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
          summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
        expr: |
          node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
        for: 5m
        labels:
          severity: warning
      # - alert: KubeletPodStartUpLatencyHigh
      #   annotations:
      #     description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
      #     summary: Kubelet Pod startup latency is too high.
      #   expr: |
      #     histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
      #   for: 15m
      #   labels:
      #     severity: warning
      - alert: KubeletClientCertificateRenewalErrors
        annotations:
          description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
          summary: Kubelet has failed to renew its client certificate.
        expr: |
          increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
        for: 15m
        labels:
          severity: warning
      - alert: KubeletServerCertificateRenewalErrors
        annotations:
          description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
          summary: Kubelet has failed to renew its server certificate.
        expr: |
          increase(kubelet_server_expiration_renew_errors[5m]) > 0
        for: 15m
        labels:
          severity: warning
      # - alert: KubeletDown
      #   annotations:
      #     description: Kubelet has disappeared from Prometheus target discovery.
      #     summary: Target disappeared from Prometheus target discovery.
      #   expr: |
      #     absent(up{job="kubelet", metrics_path="/metrics"} == 1)
      #   for: 15m
      #   labels:
      #     severity: critical
    - name: node-exporter
      rules:
      - alert: NodeFilesystemSpaceFillingUp
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
          summary: Filesystem is predicted to run out of space within the next 24 hours.
        expr: |
          (
            node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 40
          and
            predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: warning
      - alert: NodeFilesystemSpaceFillingUp
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
          summary: Filesystem is predicted to run out of space within the next 4 hours.
        expr: |
          (
            node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15
          and
            predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: critical
      - alert: NodeFilesystemAlmostOutOfSpace
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
          summary: Filesystem has less than 5% space left.
        expr: |
          (
            node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: warning
      - alert: NodeFilesystemAlmostOutOfSpace
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
          summary: Filesystem has less than 3% space left.
        expr: |
          (
            node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: critical
      - alert: NodeFilesystemFilesFillingUp
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
          summary: Filesystem is predicted to run out of inodes within the next 24 hours.
        expr: |
          (
            node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40
          and
            predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: warning
      - alert: NodeFilesystemFilesFillingUp
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
          summary: Filesystem is predicted to run out of inodes within the next 4 hours.
        expr: |
          (
            node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20
          and
            predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: critical
      - alert: NodeFilesystemAlmostOutOfFiles
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
          summary: Filesystem has less than 5% inodes left.
        expr: |
          (
            node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: warning
      - alert: NodeFilesystemAlmostOutOfFiles
        annotations:
          description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
          summary: Filesystem has less than 3% inodes left.
        expr: |
          (
            node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3
          and
            node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
          )
        for: 1h
        labels:
          severity: critical
      - alert: NodeNetworkReceiveErrs
        annotations:
          description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
          summary: Network interface is reporting many receive errors.
        expr: |
          rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
        for: 1h
        labels:
          severity: warning
      - alert: NodeNetworkTransmitErrs
        annotations:
          description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
          summary: Network interface is reporting many transmit errors.
        expr: |
          rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
        for: 1h
        labels:
          severity: warning
      - alert: NodeHighNumberConntrackEntriesUsed
        annotations:
          description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
          summary: Number of conntrack are getting close to the limit.
        expr: |
          (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
        labels:
          severity: warning
      - alert: NodeTextFileCollectorScrapeError
        annotations:
          description: Node Exporter text file collector failed to scrape.
          summary: Node Exporter text file collector failed to scrape.
        expr: |
          node_textfile_scrape_error{job="node-exporter"} == 1
        labels:
          severity: warning
      - alert: NodeClockSkewDetected
        annotations:
          description: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
          summary: Clock skew detected.
        expr: |
          (
            node_timex_offset_seconds > 0.05
          and
            deriv(node_timex_offset_seconds[5m]) >= 0
          )
          or
          (
            node_timex_offset_seconds < -0.05
          and
            deriv(node_timex_offset_seconds[5m]) <= 0
          )
        for: 10m
        labels:
          severity: warning
      - alert: NodeClockNotSynchronising
        annotations:
          description: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
          summary: Clock not synchronising.
        expr: |
          min_over_time(node_timex_sync_status[5m]) == 0
          and
          node_timex_maxerror_seconds >= 16
        for: 10m
        labels:
          severity: warning
      - alert: NodeRAIDDegraded
        annotations:
          description: RAID array '{{ $labels.device }}' on {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
          summary: RAID Array is degraded
        expr: |
          node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0
        for: 15m
        labels:
          severity: critical
      - alert: NodeRAIDDiskFailure
        annotations:
          description: At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
          summary: Failed device in RAID array
        expr: |
          node_md_disks{state="failed"} > 0
        labels:
          severity: warning
    - name: prometheus ## prometheus
      rules:
      - alert: PrometheusBadConfig
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
          summary: Failed Prometheus configuration reload.
        expr: |
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          max_over_time(prometheus_config_last_reload_successful{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) == 0
        for: 10m
        labels:
          severity: critical
      - alert: PrometheusNotificationQueueRunningFull
        annotations:
          description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
          summary: Prometheus alert notification queue predicted to run full in less than 30m.
        expr: |
          # Without min_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            predict_linear(prometheus_notifications_queue_length{job=~"prometheus-k8s|prometheus-user-workload"}[5m], 60 * 30)
          >
            min_over_time(prometheus_notifications_queue_capacity{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          )
        for: 15m
        labels:
          severity: warning
      - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
        annotations:
          description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
          summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
        expr: |
          (
            rate(prometheus_notifications_errors_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          /
            rate(prometheus_notifications_sent_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          )
          * 100
          > 1
        for: 15m
        labels:
          severity: warning
      - alert: PrometheusNotConnectedToAlertmanagers
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
          summary: Prometheus is not connected to any Alertmanagers.
        expr: |
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          max_over_time(prometheus_notifications_alertmanagers_discovered{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) < 1
        for: 10m
        labels:
          severity: warning
      - alert: PrometheusTSDBReloadsFailing
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
          summary: Prometheus has issues reloading blocks from disk.
        expr: |
          increase(prometheus_tsdb_reloads_failures_total{job=~"prometheus-k8s|prometheus-user-workload"}[3h]) > 0
        for: 4h
        labels:
          severity: warning
      - alert: PrometheusTSDBCompactionsFailing
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
          summary: Prometheus has issues compacting blocks.
        expr: |
          increase(prometheus_tsdb_compactions_failed_total{job=~"prometheus-k8s|prometheus-user-workload"}[3h]) > 0
        for: 4h
        labels:
          severity: warning
      - alert: PrometheusNotIngestingSamples
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
          summary: Prometheus is not ingesting samples.
        expr: |
          (
            rate(prometheus_tsdb_head_samples_appended_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) <= 0
          and
            (
              sum without(scrape_job) (prometheus_target_metadata_cache_entries{job=~"prometheus-k8s|prometheus-user-workload"}) > 0
            or
              sum without(rule_group) (prometheus_rule_group_rules{job=~"prometheus-k8s|prometheus-user-workload"}) > 0
            )
          )
        for: 10m
        labels:
          severity: warning
      - alert: PrometheusDuplicateTimestamps
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
          summary: Prometheus is dropping samples with duplicate timestamps.
        expr: |
          rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
        for: 1h
        labels:
          severity: warning
      - alert: PrometheusOutOfOrderTimestamps
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
          summary: Prometheus drops samples with out-of-order timestamps.
        expr: |
          rate(prometheus_target_scrapes_sample_out_of_order_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
        for: 1h
        labels:
          severity: warning
      - alert: PrometheusRemoteStorageFailures
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
          summary: Prometheus fails to send samples to remote storage.
        expr: |
          (
            rate(prometheus_remote_storage_failed_samples_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          /
            (
              rate(prometheus_remote_storage_failed_samples_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
            +
              rate(prometheus_remote_storage_succeeded_samples_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
            )
          )
          * 100
          > 1
        for: 15m
        labels:
          severity: critical
      - alert: PrometheusRemoteWriteBehind
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
          summary: Prometheus remote write is behind.
        expr: |
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          - ignoring(remote_name, url) group_right
            max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          )
          > 120
        for: 15m
        labels:
          severity: critical
      - alert: PrometheusRemoteWriteDesiredShards
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job=~"prometheus-k8s|prometheus-user-workload"}` $labels.instance | query | first | value }}.
          summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
        expr: |
          # Without max_over_time, failed scrapes could create false negatives, see
          # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          (
            max_over_time(prometheus_remote_storage_shards_desired{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          >
            max_over_time(prometheus_remote_storage_shards_max{job=~"prometheus-k8s|prometheus-user-workload"}[5m])
          )
        for: 15m
        labels:
          severity: warning
      - alert: PrometheusRuleFailures
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
          summary: Prometheus is failing rule evaluations.
        expr: |
          increase(prometheus_rule_evaluation_failures_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
        for: 15m
        labels:
          severity: critical
      - alert: PrometheusMissingRuleEvaluations
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
          summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
        expr: |
          increase(prometheus_rule_group_iterations_missed_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
      - alert: PrometheusTargetLimitHit
        annotations:
          description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
          summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
        expr: |
          increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
        for: 15m
        labels:
          severity: warning
      - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
        annotations:
          description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
          summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
        expr: |
          min without (alertmanager) (
            rate(prometheus_notifications_errors_total{job=~"prometheus-k8s|prometheus-user-workload",alertmanager!~``}[5m])
          /
            rate(prometheus_notifications_sent_total{job=~"prometheus-k8s|prometheus-user-workload",alertmanager!~``}[5m])
          )
          * 100
          > 3
        for: 15m
        labels:
          severity: critical
