#absent(): if expression inside blank is not matched, then return 1
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-alertrules
  namespace: monitoring
data:
  prometheus.rules: |-
    groups:
    - name: 'Resource-Recording' #################################################################################
      rules:
      - expr: |
          sum(rate(container_cpu_usage_seconds_total{job="kubernetes-cadvisor", image!="", container_name!=""}[5m])) by (namespace)
        record: namespace:container_cpu_usage_seconds_total:sum_rate
      - expr: |
          sum by (namespace, pod_name, container_name) (
            rate(container_cpu_usage_seconds_total{job="kubernetes-cadvisor", image!="", container_name!=""}[5m])
          )
        record: namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate
      - expr: |
          sum(container_memory_usage_bytes{job="kubernetes-cadvisor", image!="", container_name!=""}) by (namespace)
        record: namespace:container_memory_usage_bytes:sum
      - expr: |
          sum by (namespace, label_name) (
             sum(rate(container_cpu_usage_seconds_total{job="kubernetes-cadvisor", image!="", container_name!=""}[5m])) by (namespace, pod_name)
           * on (namespace, pod_name) group_left(label_name)
             label_replace(kube_pod_labels{job="kube-state-metrics"}, "pod_name", "$1", "pod", "(.*)")
          )
        record: namespace_name:container_cpu_usage_seconds_total:sum_rate
      - expr: |
          sum by (namespace, label_name) (
            sum(container_memory_usage_bytes{job="kubernetes-cadvisor",image!="", container_name!=""}) by (pod_name, namespace)
          * on (namespace, pod_name) group_left(label_name)
            label_replace(kube_pod_labels{job="kube-state-metrics"}, "pod_name", "$1", "pod", "(.*)")
          )
        record: namespace_name:container_memory_usage_bytes:sum
      - expr: |
          sum by (namespace, label_name) (
            sum(kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"}) by (namespace, pod)
          * on (namespace, pod) group_left(label_name)
            label_replace(kube_pod_labels{job="kube-state-metrics"}, "pod_name", "$1", "pod", "(.*)")
          )
        record: namespace_name:kube_pod_container_resource_requests_memory_bytes:sum
      - expr: |
          sum by (namespace, label_name) (
            sum(kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} and on(pod) kube_pod_status_scheduled{condition="true"}) by (namespace, pod)
          * on (namespace, pod) group_left(label_name)
            label_replace(kube_pod_labels{job="kube-state-metrics"}, "pod_name", "$1", "pod", "(.*)")
          )
        record: namespace_name:kube_pod_container_resource_requests_cpu_cores:sum



    - name: 'Node-Recording' #####################################################################################
      rules:
      - expr: sum(rate(node_cpu{mode!="idle",mode!="iowait"}[3m])) BY
          (instance)
        record: instance:node_cpu:rate:sum
      - expr: sum((node_filesystem_size{mountpoint="/"} - node_filesystem_free{mountpoint="/"}))
          BY (instance)
        record: instance:node_filesystem_usage:sum
      - expr: sum(rate(node_network_receive_bytes[3m])) BY (instance)
        record: instance:node_network_receive:rate:sum
      - expr: sum(rate(node_network_transmit_bytes[3m])) BY (instance)
        record: instance:node_network_transmit:rate:sum
      - expr: sum(rate(node_cpu{mode!="idle",mode!="iowait"}[5m])) WITHOUT
          (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu)
          BY (instance, cpu)) BY (instance)
        record: instance:node_cpu:ratio
      - expr: sum(rate(node_cpu{mode!="idle",mode!="iowait"}[5m]))
        record: cluster:node_cpu:sum_rate5m
      - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu)
          BY (instance, cpu))
        record: cluster:node_cpu:ratio



    - name: 'Kubernetes-absent' #####################################check whether components are running up#####
      rules:
      - alert: AlertmanagerDown #alertmanager up
        annotations:
          message: Alertmanager has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-alertmanagerdown
        expr: |
          absent(up{name="prometheus",kubernetes_namespace="monitoring"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: PrometheusDown #prometheus up
        annotations:
          message: Prometheus has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-prometheusdown
        expr: |
          absent(up{name="prometheus",kubernetes_namespace="monitoring"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: CoreDNSDown #coredns up
        annotations:
          message: CoreDNS has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-corednsdown
        expr: |
          absent(up{k8s_app="coredns"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: KubeControllerManagerDown #kube-controller-manager up
        annotations:
          message: KubeControllerManager has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown
        expr: |
          absent(up{component="kube-controller-manager"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: KubeSchedulerDown #kube-scheduler up
        annotations:
          message: KubeScheduler has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown
        expr: |
          absent(up{component="kube-scheduler"} == 1)
        for: 15m
        labels:
          severity: critical
      - alert: NodeExporterDown #node-exporter up
        annotations:
          message: NodeExporter has disappeared from Prometheus target discovery.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeexporterdown
        expr: |
          absent(up{job="node-exporter"} == 1)
        for: 15m
        labels:
          severity: critical



    - name: 'Kubernetes-Resources' #####################################check whether components are running up
      rules:
      - alert: KubeCPUOvercommit
        annotations:
          message: Cluster has overcommitted CPU resource requests for Pods and cannot
            tolerate node failure.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
        expr: |
          sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)
            /
          sum(node:node_num_cpu:sum)
            >
          (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum)
        for: 5m
        labels:
          severity: warning

      - alert: 'CPUGoOver80%'
        annotations:
          message: Cpu usage is going over 80%! Current cpu usage is {{ printf "%0.2f" $value }}%
        expr: |
          sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)
            /
          sum(node:node_num_cpu:sum)
            >
          (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum)
        for: 5m
        labels:
          severity: warning

      - alert: 'CPUGoOver90%'
        annotations:
          message: Cpu usage is going over 90%! Current cpu usage is {{ printf "%0.2f" $value }}%
        expr: |
          sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum)
            /
          sum(node:node_num_cpu:sum)
            >
          (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum)
        for: 5m
        labels:
          severity: critical


      - alert: 'MemGoOver90%'
        annotations:
          message: Memory usage is going over 90%! Current memory usage is {{ printf "%0.2f" $value }}%
        expr: |
          sum (container_memory_working_set_bytes{id="/",kubernetes_io_hostname=~"^$Node$"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~"^$Node$"}) * 100
            >
          90
        for: 5m
        labels:
          severity: warning

      - alert: 'MemGoOver95%'
        annotations:
          message: Memory usage is going over 95%! Current memory usage is {{ printf "%0.2f" $value }}%
        expr: |
          sum (container_memory_working_set_bytes{id="/",kubernetes_io_hostname=~"^$Node$"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~"^$Node$"}) * 100
            >
          90
        for: 5m
        labels:
          severity: critical


      - alert: 'FilesystemUsageGoOver95%'
        annotations:
          message: Filesystem usage is going over 95%! Current usage is {{ printf "%0.2f" $value }}%
        expr: |
          kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: critical



      - alert: KubeMemOvercommit
        annotations:
          message: Cluster has overcommitted memory resource requests for Pods and cannot
            tolerate node failure.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
        expr: |
          sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum)
            /
          sum(node_memory_MemTotal_bytes)
            >
          (count(node:node_num_cpu:sum)-1)
            /
          count(node:node_num_cpu:sum)
        for: 5m
        labels:
          severity: warning 


    
    - name: 'Kubernetes-Storage' #################################################################################
      rules:
      - alert: KubePersistentVolumeUsageCritical
        annotations:
          message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} is only {{ printf "%0.2f" $value
            }}% free.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical
        expr: |
          100 * kubelet_volume_stats_available_bytes{job="kubelet"}
            /
          kubelet_volume_stats_capacity_bytes{job="kubelet"}
            < 3
        for: 1m
        labels:
          severity: critical
      - alert: KubePersistentVolumeFullInFourDays
        annotations:
          message: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} is expected to fill up within four
            days. Currently {{ printf "%0.2f" $value }}% is available.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays
        expr: |
          100 * (
            kubelet_volume_stats_available_bytes{job="kubelet"}
              /
            kubelet_volume_stats_capacity_bytes{job="kubelet"}
          ) < 15
          and
          predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
        for: 5m
        labels:
          severity: critical
      - alert: KubePersistentVolumeErrors
        annotations:
          message: The persistent volume {{ $labels.persistentvolume }} has status {{
            $labels.phase }}.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
        expr: |
          kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: critical



    - name: 'Kubernetes-Node' ####################################################################################
      rules:
      - expr: sum(min(kube_pod_info) by (node))
        record: ':kube_pod_info_node_count:'
      - expr: |
          max(label_replace(kube_pod_info{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)")) by (node, namespace, pod)
        record: 'node_namespace_pod:kube_pod_info:'
      - expr: |
          count by (node) (sum by (node, cpu) (
            node_cpu_seconds_total{job="node-exporter"}
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          ))
        record: node:node_num_cpu:sum
      - expr: |
          1 - avg(rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m]))
        record: :node_cpu_utilisation:avg1m
      - expr: |
          1 - avg by (node) (
            rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m])
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:)
        record: node:node_cpu_utilisation:avg1m
      - expr: |
          node:node_cpu_utilisation:avg1m
            *
          node:node_num_cpu:sum
            /
          scalar(sum(node:node_num_cpu:sum))
        record: node:cluster_cpu_utilisation:ratio
      - expr: |
          sum(node_load1{job="node-exporter"})
          /
          sum(node:node_num_cpu:sum)
        record: ':node_cpu_saturation_load1:'
      - expr: |
          sum by (node) (
            node_load1{job="node-exporter"}
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
          /
          node:node_num_cpu:sum
        record: 'node:node_cpu_saturation_load1:'
      - expr: |
          1 -
          sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"})
          /
          sum(node_memory_MemTotal_bytes{job="node-exporter"})
        record: ':node_memory_utilisation:'
      - expr: |
          sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"})
        record: :node_memory_MemFreeCachedBuffers_bytes:sum
      - expr: |
          sum(node_memory_MemTotal_bytes{job="node-exporter"})
        record: :node_memory_MemTotal_bytes:sum
      - expr: |
          sum by (node) (
            (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"})
            * on (namespace, pod) group_left(node)
              node_namespace_pod:kube_pod_info:
          )
        record: node:node_memory_bytes_available:sum
      - expr: |
          sum by (node) (
            node_memory_MemTotal_bytes{job="node-exporter"}
            * on (namespace, pod) group_left(node)
              node_namespace_pod:kube_pod_info:
          )
        record: node:node_memory_bytes_total:sum
      - expr: |
          (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)
          /
          node:node_memory_bytes_total:sum
        record: node:node_memory_utilisation:ratio
      - expr: |
          (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum)
          /
          scalar(sum(node:node_memory_bytes_total:sum))
        record: node:cluster_memory_utilisation:ratio
      - expr: |
          1e3 * sum(
            (rate(node_vmstat_pgpgin{job="node-exporter"}[1m])
           + rate(node_vmstat_pgpgout{job="node-exporter"}[1m]))
          )
        record: :node_memory_swap_io_bytes:sum_rate
      - expr: |
          1 -
          sum by (node) (
            (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"})
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
          /
          sum by (node) (
            node_memory_MemTotal_bytes{job="node-exporter"}
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
        record: 'node:node_memory_utilisation:'
      - expr: |
          1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum)
        record: 'node:node_memory_utilisation_2:'
      - expr: |
          1e3 * sum by (node) (
            (rate(node_vmstat_pgpgin{job="node-exporter"}[1m])
           + rate(node_vmstat_pgpgout{job="node-exporter"}[1m]))
           * on (namespace, pod) group_left(node)
             node_namespace_pod:kube_pod_info:
          )
        record: node:node_memory_swap_io_bytes:sum_rate
      - expr: |
          avg(irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]))
        record: :node_disk_utilisation:avg_irate
      - expr: |
          avg by (node) (
            irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
        record: node:node_disk_utilisation:avg_irate
      - expr: |
          avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]))
        record: :node_disk_saturation:avg_irate
      - expr: |
          avg by (node) (
            irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
        record: node:node_disk_saturation:avg_irate
      - expr: |
          max by (namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}
          - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
          / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
        record: 'node:node_filesystem_usage:'
      - expr: |
          max by (namespace, pod, device) (node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"})
        record: 'node:node_filesystem_avail:'
      - expr: |
          sum(irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) +
          sum(irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m]))
        record: :node_net_utilisation:sum_irate
      - expr: |
          sum by (node) (
            (irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m]) +
            irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m]))
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
        record: node:node_net_utilisation:sum_irate
      - expr: |
          sum(irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m])) +
          sum(irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m]))
        record: :node_net_saturation:sum_irate
      - expr: |
          sum by (node) (
            (irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m]) +
            irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m]))
          * on (namespace, pod) group_left(node)
            node_namespace_pod:kube_pod_info:
          )
        record: node:node_net_saturation:sum_irate
      - expr: |
          max(
            max(
              kube_pod_info{job="kube-state-metrics", host_ip!=""}
            ) by (node, host_ip)
            * on (host_ip) group_right (node)
            label_replace(
              (max(node_filesystem_files{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*"
            )
          ) by (node)
        record: 'node:node_inodes_total:'
      - expr: |
          max(
            max(
              kube_pod_info{job="kube-state-metrics", host_ip!=""}
            ) by (node, host_ip)
            * on (host_ip) group_right (node)
            label_replace(
              (max(node_filesystem_files_free{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*"
            )
          ) by (node)
        record: 'node:node_inodes_free:'



    - name: 'Micro-Service' ######################################################################################
      rules:
      - alert: 'RequestSkyrocket'
        annotations:
          message: '{{ $labels.name }} requests are above 200!'
        expr: |
          sum by (name) (rate(request_duration_seconds_count{route!="metrics",endpoint_namespace="sock-shop"}[1m])) * 100 
            >=
          200
        for: 5m
        labels:
          severity: warning
      - alert: 'LatencySkyrocket'
        annotations:
          message: '{{ $labels.name }} latency is above 0.008!'
        expr: |
          sum(rate(request_duration_seconds_sum{endpoint_namespace="sock-shop"}[1m])) by (name) / sum(rate(request_duration_seconds_count{endpoint_namespace="sock-shop"}[1m])) by (name)
            >
          0.007
        for: 5m
        labels:
          seveity: warning

#      - alert: CPUThrottlingHigh
#        annotations:
#          message: '{{ printf "%0.0f" $value }}% throttling of CPU in namespace {{ $labels.namespace
#            }} for container {{ $labels.container_name }} in pod {{ $labels.pod_name
#            }}.'
#          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
#        expr: "100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!=\"\",
#          }[5m])) by (container_name, pod_name, namespace)\n  /\nsum(increase(container_cpu_cfs_periods_total{}[5m]))
#          by (container_name, pod_name, namespace)\n  > 25 \n"
#        for: 15m
#        labels:
#          severity: warning
#
