apiVersion: v1
data:
  alert.rules: |
    ALERT PodsMissing
      IF app:up:ratio{app!=""} < 0.9
      FOR 10m
      LABELS { severity="page" }
      ANNOTATIONS {
        summary = "Pods missing from {{`{{`}} $labels.app {{`}}`}}",
        description = "Pods missing from {{`{{`}} $labels.app {{`}}`}}: {{`{{`}} $value {{`}}`}}",
        help = "Alerts when pods are missing.",
      }

    ALERT NoPodsRunning
      IF app:up:ratio{app!=""} < 0.1
      FOR 2m
      LABELS { severity="page" }
      ANNOTATIONS {
        summary = "No pods are running for {{`{{`}} $labels.app {{`}}`}}",
        description = "No pods are running for {{`{{`}} $labels.app {{`}}`}}: {{`{{`}} $value {{`}}`}}",
        help = "Alerts when no pods are running for a service.",
      }

    ALERT ProdPageLoadLatency
      IF histogram_quantile(0.9, sum(rate(src_http_request_duration_seconds_bucket{job="sourcegraph-frontend",route!="xlang",route!="lsp"}[10m])) by (le)) > 20
      LABELS { severity="page" }
      ANNOTATIONS {
        summary = "High page load latency",
        description = "Page load latency > 20s (90th percentile over all routes; current value: {{`{{`}}$value{{`}}`}}s)",
        help = "Alerts when the page load latency is too high.",
      }

    ALERT GoroutineLeak
      IF go_goroutines >= 10000
      FOR 10m
      ANNOTATIONS {
        summary = "Excessive number of goroutines",
        description = "{{`{{`}} $labels.app {{`}}`}} has more than 10k goroutines. This is probably a regression causing a goroutine leak",
        help = "Alerts when a service has excessive running goroutines.",
      }

    ALERT FSINodesRemainingLow
      IF sum by (instance) (container_fs_inodes_total{pod_name!=""}) > 3000000
      LABELS { severity = "page" }
      ANNOTATIONS {
        summary = "{{`{{`}}$labels.instance{{`}}`}} remaining fs inodes is low",
        description = "{{`{{`}}$labels.instance{{`}}`}} is using {{`{{`}}humanize $value{{`}}`}} inodes",
        help = "Alerts when a node's remaining FS inodes are low.",
      }

    ALERT DiskSpaceLow
      IF node:k8snode_filesystem_avail_bytes:ratio < 0.10
      ANNOTATIONS {
        summary = "{{`{{`}}$labels.exported_name{{`}}`}} has less than 10% available disk space",
        help = "Alerts when a node has less than 10% available disk space.",
      }

    ALERT DiskSpaceLowCritical
      IF node:k8snode_filesystem_avail_bytes:ratio{exported_name=~".*prod.*"} < 0.05
      LABELS { severity="page" }
      ANNOTATIONS {
        summary = "Critical! {{`{{`}}$labels.exported_name{{`}}`}} has less than 5% available disk space",
        help = "Alerts when a node has less than 5% available disk space.",
      }

    ALERT SearcherErrorRatioTooHigh
      IF searcher_errors:ratio10m > 0.1
      FOR 20m
      ANNOTATIONS {
        summary = "Error ratio exceeds 10%",
        help = "Alerts when the search service has more than 10% of requests failing.",
      }

    # TODO(sqs): enable this after we have tested it in prod, to avoid needless alerts
    #
    # ALERT SymbolsErrorRatioTooHigh
    #   IF symbols_errors:ratio10m > 0.1
    #   FOR 20m
    #   ANNOTATIONS {
    #     summary = "Error ratio exceeds 10%",
    #     help = "Alerts when the symbols service has more than 10% of requests failing.",
    #   }

    # NOTE(beyang): this should no longer be part of the default metrics, because not all DC instances have Go code intelligence
    #
    # ALERT XLangAbsent
    #   IF absent(prod:xlang_requests:rate5m{mode="go"})
    #   FOR 1m
    #   LABELS { severity="page" }
    #   ANNOTATIONS {
    #     summary = "{{`{{`}} $labels.mode {{`}}`}} is not running",
    #     help = "Alerts when the Go language server is not running.",
    #     readme = "enterprise",
    #   }

    # http_response_size_bytes is measured at the Prometheus clients, not at the server.
    ALERT PrometheusMetricsBloat
      IF http_response_size_bytes{handler="prometheus", quantile="0.5", job!="kubernetes-nodes", job!="kubernetes-apiservers"} > 20000
      ANNOTATIONS {
        summary = "{{`{{`}}$labels.job{{`}}`}} in {{`{{`}}$labels.ns{{`}}`}} is probably leaking metrics (unbounded attribute)",
        help = "Alerts when a service is probably leaking metrics (unbounded attribute).",
      }
  apdex.rules: |+
    # Generated with apdex.py
    # Do not manually edit

    # Rate 15m
    prod:src_xlang_request_duration_seconds:apdex_02_08_rate15m =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.2",success="true"}[15m])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.8",success="true"}[15m])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[15m])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_1_5_rate15m =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="1",success="true"}[15m])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="5",success="true"}[15m])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[15m])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_2_10_rate15m =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="2",success="true"}[15m])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="10",success="true"}[15m])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[15m])) by (mode, method)


    # Rate 1h
    prod:src_xlang_request_duration_seconds:apdex_02_08_rate1h =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.2",success="true"}[1h])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.8",success="true"}[1h])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1h])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_1_5_rate1h =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="1",success="true"}[1h])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="5",success="true"}[1h])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1h])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_2_10_rate1h =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="2",success="true"}[1h])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="10",success="true"}[1h])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1h])) by (mode, method)


    # Rate 1d
    prod:src_xlang_request_duration_seconds:apdex_02_08_rate1d =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.2",success="true"}[1d])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="0.8",success="true"}[1d])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1d])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_1_5_rate1d =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="1",success="true"}[1d])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="5",success="true"}[1d])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1d])) by (mode, method)

    prod:src_xlang_request_duration_seconds:apdex_2_10_rate1d =
      (
        sum(rate(src_xlang_request_duration_seconds_bucket{le="2",success="true"}[1d])) by (mode, method)
      +
        sum(rate(src_xlang_request_duration_seconds_bucket{le="10",success="true"}[1d])) by (mode, method)
      ) / 2 / sum(rate(src_xlang_request_duration_seconds_count[1d])) by (mode, method)


  extra.rules: ""
  node.rules: |
    node:container_cpu_usage_seconds_total:ratio_rate5m =
      sum by (instance) (rate(container_cpu_usage_seconds_total{kubernetes_pod_name=""}[5m]))
      /
      max by (instance) (machine_cpu_cores)

    task:container_memory_usage_bytes:max = max by (namespace, container_name)(container_memory_usage_bytes{container_name!=""})
    task:container_cpu_usage_seconds_total:sum = sum by (id, namespace, container_name) (irate(container_cpu_usage_seconds_total{container_name!=""}[1m]))

    node:k8snode_filesystem_avail_bytes:ratio =
      min by (exported_name) (k8snode_filesystem_avail_bytes / k8snode_filesystem_size_bytes)
  prometheus.yml: |
    ---

    global:
      scrape_interval:     30s
      evaluation_interval: 30s

    rule_files:
      - 'node.rules'
      - 'sourcegraph.rules'
      - 'xlang.rules'
      - 'apdex.rules'
      - 'searcher.rules'
      - 'symbols.rules'
      - 'alert.rules'
      - 'extra.rules'

    # A scrape configuration for running Prometheus on a Kubernetes cluster.
    # This uses separate scrape configs for cluster components (i.e. API server, node)
    # and services to allow each to use different authentication configs.
    #
    # Kubernetes labels will be added as Prometheus labels on metrics via the
    # `labelmap` relabeling action.

    # Scrape config for API servers.
    #
    # Kubernetes exposes API servers as endpoints to the default/kubernetes
    # service so this uses `endpoints` role and uses relabelling to only keep
    # the endpoints associated with the default/kubernetes service using the
    # default named port `https`. This works for single API server deployments as
    # well as HA API server deployments.
    scrape_configs:
    - job_name: 'kubernetes-apiservers'

      kubernetes_sd_configs:
      - role: endpoints

      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https

      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration because discovery & scraping are two separate concerns in
      # Prometheus. The discovery auth config is automatic if Prometheus runs inside
      # the cluster. Otherwise, more config options have to be provided within the
      # <kubernetes_sd_config>.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        # insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      # Keep only the default/kubernetes service endpoints for the https port. This
      # will add targets for each API server which Kubernetes adds an endpoint to
      # the default/kubernetes service.
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

    - job_name: 'kubernetes-nodes'

      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https

      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration because discovery & scraping are two separate concerns in
      # Prometheus. The discovery auth config is automatic if Prometheus runs inside
      # the cluster. Otherwise, more config options have to be provided within the
      # <kubernetes_sd_config>.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:
      - role: node

      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics

    # Scrape config for service endpoints.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
    # to set this to `https` & most likely set the `tls_config` of the scrape config.
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
    # service then set this appropriately.
    - job_name: 'kubernetes-service-endpoints'

      kubernetes_sd_configs:
      - role: endpoints

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        # Sourcegraph specific customization. We want a more convenient to type label.
        # target_label: kubernetes_namespace
        target_label: ns
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name
      # Sourcegraph specific customization. We want a nicer name for job
      - source_labels: [app]
        action: replace
        target_label: job
      # Sourcegraph specific customization. We want a nicer name for instance
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: instance

    # Example scrape config for probing services via the Blackbox Exporter.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/probe`: Only probe services that have a value of `true`
    - job_name: 'kubernetes-services'

      metrics_path: /probe
      params:
        module: [http_2xx]

      kubernetes_sd_configs:
      - role: service

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        # Sourcegraph specific customization. We want a more convenient to type label.
        # target_label: kubernetes_namespace
        target_label: ns
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name

    # Example scrape config for pods
    #
    # The relabeling allows the actual pod scrape endpoint to be configured via the
    # following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`.
    - job_name: 'kubernetes-pods'

      kubernetes_sd_configs:
      - role: pod

      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+):(?:\d+);(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        # Sourcegraph specific customization. We want a more convenient to type label.
        # target_label: kubernetes_namespace
        target_label: ns
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name
  searcher.rules: |
    ##########
    # Errors #
    ##########

    searcher_requests:rate10m = sum by (instance)(rate(searcher_service_request_total[10m]))
    searcher_errors:rate10m   = sum by (instance)(rate(searcher_service_request_total{code!="200",code!="canceled"}[10m]))
    searcher_errors:ratio10m = sum(searcher_errors:rate10m) / sum(searcher_requests:rate10m)
  sourcegraph.rules: |
    # This files contains recording rules targeted at the sourcegraph-frontend app

    # Duration (5m)
    task:src_http_request_duration_seconds_bucket:rate5m = rate(src_http_request_duration_seconds_bucket{job=~"sourcegraph-.*"}[5m])
    route:src_http_request_duration_seconds_bucket:rate5m = sum by (route, ns, le)(task:src_http_request_duration_seconds_bucket:rate5m)
    job:src_http_request_duration_seconds_bucket:rate5m = sum by (ns, le)(task:src_http_request_duration_seconds_bucket:rate5m)
    prod:src_http_request_duration_seconds_bucket:rate5m = sum by (le)(task:src_http_request_duration_seconds_bucket:rate5m)

    # Counts (5m)
    task:src_http_request_count:rate5m = rate(src_http_request_duration_seconds_count{job=~"sourcegraph-.*"}[5m])
    route:src_http_request_count:rate5m = sum by (route, code, ns)(task:src_http_request_count:rate5m)
    job:src_http_request_count:rate5m = sum by (code, ns)(task:src_http_request_count:rate5m)
    prod:src_http_request_count:rate5m = sum by (code)(task:src_http_request_count:rate5m)

    # Duration (30m)
    task:src_http_request_duration_seconds_bucket:rate30m = rate(src_http_request_duration_seconds_bucket{job=~"sourcegraph-.*"}[30m])
    route:src_http_request_duration_seconds_bucket:rate30m = sum by (route, ns, le)(task:src_http_request_duration_seconds_bucket:rate30m)
    job:src_http_request_duration_seconds_bucket:rate30m = sum by (ns, le)(task:src_http_request_duration_seconds_bucket:rate30m)
    prod:src_http_request_duration_seconds_bucket:rate30m = sum by (le)(task:src_http_request_duration_seconds_bucket:rate30m)

    # Counts (30m)
    task:src_http_request_count:rate30m = rate(src_http_request_duration_seconds_count{job=~"sourcegraph-.*"}[30m])
    route:src_http_request_count:rate30m = sum by (route, code, ns)(task:src_http_request_count:rate30m)
    job:src_http_request_count:rate30m = sum by (code, ns)(task:src_http_request_count:rate30m)
    prod:src_http_request_count:rate30m = sum by (code)(task:src_http_request_count:rate30m)


    # Perf targets are over a day
    prod:src_http_request_duration_seconds_bucket:rate1d = sum by (route, le)(rate(src_http_request_duration_seconds_bucket{job=~"sourcegraph-.*"}[1d]))


    # Measure uptime of services
    app:up:sum = sum by (app)(up)
    app:up:count = count by (app)(up)
    app:up:ratio =
      app:up:sum
        / on (app)
      app:up:count
  symbols.rules: |
    ##########
    # Errors #
    ##########

    symbols_requests:rate10m = sum by (instance)(rate(symbols_service_request_total[10m]))
    symbols_errors:rate10m   = sum by (instance)(rate(symbols_service_request_total{code!="200",code!="canceled"}[10m]))
    symbols_errors:ratio10m = sum(symbols_errors:rate10m) / sum(symbols_requests:rate10m)
  xlang.rules: |
    prod:xlang_requests:rate5m = sum by (mode)(rate(src_xlang_request_duration_seconds_count[5m]))
    prod:xlang_errors:rate5m   = sum by (mode)(rate(src_xlang_request_duration_seconds_count{success="false"}[5m]))
    prod:xlang_errors:ratio5m  = prod:xlang_errors:rate5m / prod:xlang_requests:rate5m
    prod:src_xlang_request_duration_seconds_bucket:rate1d =
      sum by (le, mode, method)(rate(src_xlang_request_duration_seconds_bucket{success="true"}[1d]))
kind: ConfigMap
metadata:
  labels:
    deploy: prometheus
  name: prometheus
