apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus
  namespace: ethereum-2
  labels:
    app.kubernetes.io/name: prometheus
data:
  alerting_rules.yml: |
    groups:
    - name: generic
      rules:
      - alert: PodDown
        expr: up == 0
        for: 5m
        labels:
          severity: page
        annotations:
          summary: "Pod {{ $labels.kubernetes_pod_name }} down"
          description: "{{ $labels.kubernetes_pod_name }} of application {{ $labels.app_kubernetes_io_name }} has been down for more than 5 minutes."

      - alert: ContainerNotRunning
        expr: |
          kube_pod_container_status_running != 1
        for: 5m
        annotations:
          summary: "Container {{ $labels.container }} not running"
          description: "{{ $labels.container }} of Pod {{ $labels.pod }} has been down for more than 5 minutes."

    - name: kube-state-metrics
      rules:
      - alert: KubeStateMetricsListErrors
        annotations:
          message: kube-state-metrics is experiencing errors at an elevated rate in list
            operations. This is likely causing it to not be able to expose metrics about
            Kubernetes objects correctly or at all.
        expr: |
          (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical
      - alert: KubeStateMetricsWatchErrors
        annotations:
          message: kube-state-metrics is experiencing errors at an elevated rate in watch
            operations. This is likely causing it to not be able to expose metrics about
            Kubernetes objects correctly or at all.
        expr: |
          (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
            /
          sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])))
          > 0.01
        for: 15m
        labels:
          severity: critical


  prometheus.yml: |
    rule_files:
    - /etc/prometheus/alerting_rules.yml

    scrape_configs:

    - job_name: kubernetes-cadvisor
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

    - job_name: kubernetes-pods
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name
