apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-alert-rules
  namespace: kube-system
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: EnsureExists
data:
  rules.yml: |
    groups:
    - name: kube-state-metrics
      rules:
      - alert: KubeStateMetricsListErrors
        annotations:
          message: kube-state-metrics is experiencing errors at an elevated rate in list
            operations. This is likely causing it to not be able to expose metrics about
            Kubernetes objects correctly or at all.
        expr: |
          (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
      - alert: KubeStateMetricsWatchErrors
        annotations:
          message: kube-state-metrics is experiencing errors at an elevated rate in watch
            operations. This is likely causing it to not be able to expose metrics about
            Kubernetes objects correctly or at all.
        expr: |
          (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
    - name: node
      rules:
      - alert: InstanceDown
        expr: up == 0
        for: 5m
        labels:
          severity: error
        annotations:
          summary: "Instance {{ $labels.instance }} down"
          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

      - alert: CPULoad
        expr: node_load1 / (count without (cpu, mode) (node_cpu_seconds_total{mode="system"})) > 2
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "CPU load (instance {{ $labels.instance }})"
          description: "CPU load (15m) is high\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: CPUContextSwitching
        expr: rate(node_context_switches_total[5m]) > 35000
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Context switching (instance {{ $labels.instance }})"
          description: "Context switching is growing on node (> 35000 / s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: OutOfMemory
        expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Out of memory (instance {{ $labels.instance }})"
          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualNetworkThroughputIn
        expr: sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual network throughput in (instance {{ $labels.instance }})"
          description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualNetworkThroughputOut
        expr: sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual network throughput out (instance {{ $labels.instance }})"
          description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualDiskReadRate
        expr: sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual disk read rate (instance {{ $labels.instance }})"
          description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualDiskWriteRate
        expr: sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual disk write rate (instance {{ $labels.instance }})"
          description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: OutOfDiskSpace
        expr: node_filesystem_free_bytes{mountpoint ="/rootfs"} / node_filesystem_size_bytes{mountpoint ="/rootfs"} * 100 < 10
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Out of disk space (instance {{ $labels.instance }})"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: OutOfInodes
        expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Out of inodes (instance {{ $labels.instance }})"
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualDiskReadLatency
        expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual disk read latency (instance {{ $labels.instance }})"
          description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: UnusualDiskWriteLatency
        expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Unusual disk write latency (instance {{ $labels.instance }})"
          description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: NodeHasSwap
        expr: node_memory_SwapTotal_bytes > 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Node has swap (instance {{ $labels.instance }})"
          description: "Node has swap\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
    - name: pod
      rules:
      - alert: PodCPU
        expr: sum(rate(container_cpu_usage_seconds_total{image!=""}[1m])) by (pod_name, namespace) / (sum(container_spec_cpu_quota{image!=""}/100000) by (pod_name, namespace)) * 100 > 50
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "CPU usage (Pod {{ $labels.instance }})"
          description: "CPU usage (10m) is high(>20%)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: PodMemory
        expr: sum(container_memory_rss{image!=""}) by(pod_name, namespace) / sum(container_spec_memory_limit_bytes{image!=""}) by(pod_name, namespace) * 100 != +inf > 75
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Memory usage (Pod {{ $labels.instance }})"
          description: "Memory usage (10m) is high(> 70%)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
    - name: exporter
      rules:
      - alert: ExporterDown
        expr: up == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Exporter down (instance {{ $labels.instance }})"
          description: "Prometheus exporter down\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
    - name: container
      rules:
      - alert: ContainerKilled
        expr: time() - container_last_seen > 60
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Container killed (instance {{ $labels.instance }})"
          description: "A container has disappeared\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
    - name: ZookeeperStatus
      rules:
      - alert: ZookeeperTooManyOutstandingRequests
        expr: avg(zk_outstanding_requests) by (instance) > 10
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper 请求发生堆积，数量大于10。"
      - alert: ZookeeperTooManyPendingSyncs
        expr: avg(zk_pending_syncs) by (instance) > 10
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper 当前处于阻塞中的 sync 过多"
      - alert: ZookeeperAVGLatencyTooHigh
        expr: avg(zk_avg_latency) by (instance) > 10
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper 平均响应延迟过高"
      - alert: ZookeeperTooManyOpenFileDescripor
        expr: zk_open_file_descriptor_count > zk_max_file_descriptor_count * 0.85
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper 打开文件描述符数大于系统设定的大小"
      - alert: ZookeeperDown
        expr: zk_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper is DOWN"
      - alert: ZookeeperLostLeader
        expr: absent(zk_server_state{state="leader"}) != 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} "
          description: "Zookeeper LOST Leader"