---
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    app: prometheus
    component: server
    repo: ops-kube-monitoring
    deployed-with: kubectl
  name: prometheus-server-cm
  namespace: kube-system
data:
  prometheus.yml: |-
    alerting:
      alertmanagers:
        - static_configs:
          - targets:
            - prometheus-alertmanager-svc
    rule_files:
      - /etc/config/rules
      - /etc/config/alerts
      - /etc/config/*.alerts

    scrape_configs:
      - job_name: prometheus
        static_configs:
          - targets:
            - localhost:9090

      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
          - role: endpoints

        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
            action: keep
            regex: default;kubernetes;https

      - job_name: 'kubernetes-nodes'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        kubernetes_sd_configs:
          - role: node
        relabel_configs:
          - action: labelmap
            regex: __meta_kubernetes_node_label_(.+)
          - target_label: __address__
            replacement: kubernetes.default.svc:443
          - source_labels: [__meta_kubernetes_node_name]
            regex: (.+)
            target_label: __metrics_path__
            replacement: /api/v1/nodes/${1}/proxy/metrics

      - job_name: 'kubernetes-service-endpoints'
        kubernetes_sd_configs:
          - role: endpoints
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
            action: replace
            target_label: __scheme__
            regex: (https?)
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
            action: replace
            target_label: __address__
            regex: (.+)(?::\d+);(\d+)
            replacement: $1:$2
          - action: labelmap
            regex: __meta_kubernetes_service_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: kubernetes_namespace
          - source_labels: [__meta_kubernetes_service_name]
            action: replace
            target_label: kubernetes_name
          - source_labels: [__meta_kubernetes_pod_node_name]
            action: replace
            target_label: instance

      - job_name: 'prometheus-pushgateway'
        honor_labels: true
        kubernetes_sd_configs:
          - role: service
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
            action: keep
            regex: pushgateway

      - job_name: 'kubernetes-services'
        metrics_path: /probe
        params:
          module: [http_2xx]
        kubernetes_sd_configs:
          - role: service
        relabel_configs:
          - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
            action: keep
            regex: true
          - source_labels: [__address__]
            target_label: __param_target
          - target_label: __address__
            replacement: blackbox
          - source_labels: [__param_target]
            target_label: instance
          - action: labelmap
            regex: __meta_kubernetes_service_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            target_label: kubernetes_namespace
          - source_labels: [__meta_kubernetes_service_name]
            target_label: kubernetes_name

      - job_name: 'kubernetes-pods'

        kubernetes_sd_configs:
          - role: pod

        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
            action: keep
            regex: true
          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
            action: replace
            target_label: __metrics_path__
            regex: (.+)
          - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
            action: replace
            regex: (.+):(?:\d+);(\d+)
            replacement: ${1}:${2}
            target_label: __address__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: kubernetes_namespace
          - source_labels: [__meta_kubernetes_pod_name]
            action: replace
            target_label: kubernetes_pod_name

      - job_name: 'kubernetes-apiserver-pods'
        kubernetes_sd_configs:
          - role: pod
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_k8s_app, __meta_kubernetes_pod_container_port_name]
            action: keep
            regex: kube-system;kube-apiserver;https
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - source_labels: [__meta_kubernetes_namespace]
            action: replace
            target_label: kubernetes_namespace
          - source_labels: [__meta_kubernetes_pod_name]
            action: replace
            target_label: kubernetes_pod_name


      - job_name: 'kubernetes-cadvisor'
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
          insecure_skip_verify: true
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

        kubernetes_sd_configs:
        - role: node
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)
        - target_label: __address__
          replacement: kubernetes.default.svc:443
        - source_labels: [__meta_kubernetes_node_name]
          regex: (.+)
          target_label: __metrics_path__
          replacement: /api/v1/nodes/${1}:4194/proxy/metrics

  alerts: |-
    groups:
    - name: KubeSystemAlerts
      rules:
      - alert: KubeSystemPodsTooManyRestarts
        expr: kube_pod_labels and ON(pod, namespace) increase(kube_pod_container_status_restarts_total{namespace="kube-system"}[7m] OFFSET 3m) > 2
        for: 1m
        labels:
          container: '{{ "{{ $labels.container }}" }}'
          pod: '{{ "{{ $labels.pod }}" }}'
          support: '{{ "{{ if eq $labels.label_support `businessHours`}}businessHours{{else}}afterHours{{end}}" }}'
        annotations:
          description: 'Pod: {{ "{{ $labels.pod }}" }} has restarted too many times ({{ "{{ $value }}" }}).'
          formattedSummary: "Pod: \n\t '{{ "{{ $labels.pod }}" }}'"
          labelSupport: '{{ "{{ $labels.label_support }}" }}'
          summary: 'Pod: {{ "{{ $labels.pod }}" }}'
          title: 'The following kube-system pods have restarted too many times:'
      - alert: IngressTooManyWaiting
        expr: (sum(avg_over_time(nginx_connections_total{type="waiting"}[30m])) / count(kube_pod_container_status_ready{namespace="kube-system",pod=~".*ingress.*"})) > 50
        labels:
          support: afterHours
        annotations:
          description: The number of ingress nginx connections in waiting state is too many.
          formattedSummary: "Waiting per pod:\t{{ "{{ $value }}" }}"
          labelSupport: afterHours
          summary: 'Waiting per pod: {{ "{{ $value }}" }}'
          title: 'Ingress: too many waiting connections per pod ({{ "{{ $value }}" }})'
      - alert: KubeNodeNotReady
        expr: kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 1h
        labels:
          severity: businessHours
        annotations:
          description: The Kubelet on {{ "{{ $labels.node }}" }} has not checked in with the API,
            or has set itself to NotReady, for more than an hour
          summary: Node {{ "{{ $labels.node }}" }} status is NotReady
