apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus
  namespace: prometheus
data:
  prometheus.rules.yml: |
    groups:
      - name: example
        rules:
          - record: job_service:rpc_durations_seconds_count:avg_rate5m
            expr: avg(rate(rpc_durations_seconds_count[6m])) by (job, service)
      - name: dms
        rules:
        - alert: DeadMansSwitch
          annotations:
            description: This is a DeadMansSwitch meant to ensure that the entire Alerting
              pipeline is functional.
            summary: Alerting DeadMansSwitch
          expr: vector(1)
          labels:
            severity: none
      - name: CPU high on node
        rules:
          - alert: X CPU high on node X
            annotations:
              description: Damn it
            expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{job="node_exporter",mode="idle"}[1m])) * 100) > 40
            for: 5m
            labels:
              severity: high

      - name: Memory usage high on node
        rules:
          - alert: Memory usage high on node
            annotations:
              description: Another description
            expr: sum by (instance) (1 - avg_over_time(node_memory_MemAvailable_bytes{job="node_exporter"}[5m]) / avg_over_time(node_memory_MemTotal_bytes{job="node_exporter"}[5m])) * 100 > 80
            for: 1m
            labels:
              severity: high

      - name: Disk space low on node
        rules:
          - alert: Disk space low on node
            annotations:
              description: Another description
            expr: sum by (instance) (avg_over_time(node_filesystem_avail_bytes{mountpoint="/"}[5m]) / avg_over_time(node_filesystem_size_bytes{mountpoint="/"}[5m])) * 100 < 50
            for: 1m
            labels:
              severity: high

      #- name: Capacity issues
        #rules:
          #- alert: Unschedulable Pods
            #annotations:
              #description: Some pods could not be scheduled, most likely due to capacity concerns.
              #summary: Unschedulable Pods.
            #expr: avg_over_time(kube_pod_status_scheduled{condition="false"}[5m]) == 1
            #labels:
              #severity: high

  prometheus.yml: |
    global:
      scrape_interval: 15s
      # Attach these labels to any time series or alerts when communicating with
      # external systems (federation, remote storage, Alertmanager).
      external_labels:
        monitor: "foobar"

    rule_files:
      - "prometheus.rules.yml"

    alerting:
      alertmanagers:
      - scheme: http
        static_configs:
        - targets:
           - "alertmanager:9093"

    scrape_configs:
      # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
      - job_name: "prometheus"
        # Override the global default and scrape targets from this job every 5 seconds.
        scrape_interval: 5s
        static_configs:
          - targets: ["127.0.0.1:9090"]

      # From a combination of:
      # https://stackoverflow.com/a/56965672
      # https://github.com/prometheus/prometheus/blob/release-2.13/documentation/examples/prometheus-kubernetes.yml
      - job_name: "node_exporter"
        metrics_path: /metrics
        scheme: http
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names:
                - kube-system
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_endpoints_name, __meta_kubernetes_endpoint_port_name]
            action: keep
            regex: kube-system;prometheus-node-exporter;metrics

      - job_name: "kube-state-metrics"
        static_configs:
          - targets: ["kube-state-metrics.kube-system:8080"]

      - job_name: "kubernetes-api-servers"
        kubernetes_sd_configs:
          - role: endpoints
            api_server: "https://kubernetes.default.svc.cluster.local"
            tls_config:
              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
            action: keep
            regex: default;kubernetes;https

    #  - job_name: "example-random"
    #    # Override the global default and scrape targets from this job every 5 seconds
    #    scrape_interval: 5s
    #    static_configs:
    #      - targets: ["random-8080:8080", "random-8081:8081"]
    #        labels:
    #          group: "production"
    #      - targets: ["random-8082:8082"]
    #        labels:
    #          groups: "canary"
