apiVersion: v1
data:
  rule.json: |-
    {
      "annotations": {
        "list": [

        ]
      },
      "editable": true,
      "gnetId": null,
      "graphTooltip": 0,
      "hideControls": false,
      "links": [

      ],
      "refresh": "10s",
      "rows": [
        {
          "collapse": false,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 1,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job, strategy) (rate(prometheus_rule_evaluations_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ strategy }}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rule Group Evaluations",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 2,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job, strategy) (increase(prometheus_rule_group_iterations_missed_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ strategy }}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rule Group Evaluations Missed",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 3,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "(\n  max by(namespace, job, rule_group) (prometheus_rule_group_last_duration_seconds{namespace=\"$namespace\", job=\"$job\"})\n  >\n  sum by(namespace, job, rule_group) (prometheus_rule_group_interval_seconds{namespace=\"$namespace\", job=\"$job\"})\n)\n",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{ rule_group }}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rule Group Evlauations Too Slow",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "Rule Group Evaluations",
          "titleSize": "h6"
        },
        {
          "collapse": false,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of dropped alerts.",
              "fill": 1,
              "id": 4,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job, alertmanager) (rate(thanos_alert_sender_alerts_dropped_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{alertmanager}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Dropped Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of alerts that successfully sent to alert manager.",
              "fill": 10,
              "id": 5,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job, alertmanager) (rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{alertmanager}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Sent Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {
                "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of sent alerts.",
              "fill": 10,
              "id": 6,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 3,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job) (rate(thanos_alert_sender_errors_total{namespace=\"$namespace\", job=\"$job\"}[$interval])) / sum by (namespace, job) (rate(thanos_alert_sender_alerts_sent_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "error",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Sent Errors",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to send alerts to alert manager.",
              "fill": 1,
              "id": 7,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "p99",
                  "color": "#FA6400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p90",
                  "color": "#E0B400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p50",
                  "color": "#37872D",
                  "fill": 10,
                  "fillGradient": 0
                }
              ],
              "spaceLength": 10,
              "span": 3,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.50, sum by (namespace, job, le) (rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\", job=\"$job\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p50 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.90, sum by (namespace, job, le) (rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\", job=\"$job\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p90 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.99, sum by (namespace, job, le) (rate(thanos_alert_sender_latency_seconds_bucket{namespace=\"$namespace\", job=\"$job\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p99 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Sent Duration",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "Alert Sent",
          "titleSize": "h6"
        },
        {
          "collapse": false,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of queued alerts.",
              "fill": 1,
              "id": 8,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 6,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job) (rate(thanos_alert_queue_alerts_dropped_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{job}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Push Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {
                "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of dropped alerts compared to the total number of queued alerts.",
              "fill": 10,
              "id": 9,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 6,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job) (rate(thanos_alert_queue_alerts_dropped_total{namespace=\"$namespace\", job=\"$job\"}[$interval])) / sum by (namespace, job) (rate(thanos_alert_queue_alerts_pushed_total{namespace=\"$namespace\", job=\"$job\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "error",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Drop Ratio",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "Alert Queue",
          "titleSize": "h6"
        },
        {
          "collapse": false,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of handled Unary gRPC requests.",
              "fill": 10,
              "id": 10,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "/Aborted/",
                  "color": "#EAB839"
                },
                {
                  "alias": "/AlreadyExists/",
                  "color": "#37872D"
                },
                {
                  "alias": "/FailedPrecondition/",
                  "color": "#E0B400"
                },
                {
                  "alias": "/Unimplemented/",
                  "color": "#E0B400"
                },
                {
                  "alias": "/InvalidArgument/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/NotFound/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/PermissionDenied/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/Unauthenticated/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/Canceled/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/DataLoss/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/DeadlineExceeded/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Internal/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/OutOfRange/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/ResourceExhausted/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Unavailable/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Unknown/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/OK/",
                  "color": "#37872D"
                },
                {
                  "alias": "error",
                  "color": "#C4162A"
                }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace,  job, grpc_method, grpc_code) (rate(grpc_server_handled_total{namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{namespace}} {{job}} {{grpc_method}} {{grpc_code}}",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {
                "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests.",
              "fill": 10,
              "id": 11,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval])) / sum by (namespace, job) (rate(grpc_server_handled_total{namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "error",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests, in quantiles.",
              "fill": 1,
              "id": 12,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "p99",
                  "color": "#FA6400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p90",
                  "color": "#E0B400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p50",
                  "color": "#37872D",
                  "fill": 10,
                  "fillGradient": 0
                }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.50, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p50 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.90, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p90 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.99, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"unary\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p99 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "gRPC (Unary)",
          "titleSize": "h6"
        },
        {
          "collapse": false,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows rate of handled Streamed gRPC requests.",
              "fill": 10,
              "id": 13,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "/Aborted/",
                  "color": "#EAB839"
                },
                {
                  "alias": "/AlreadyExists/",
                  "color": "#37872D"
                },
                {
                  "alias": "/FailedPrecondition/",
                  "color": "#E0B400"
                },
                {
                  "alias": "/Unimplemented/",
                  "color": "#E0B400"
                },
                {
                  "alias": "/InvalidArgument/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/NotFound/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/PermissionDenied/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/Unauthenticated/",
                  "color": "#1F60C4"
                },
                {
                  "alias": "/Canceled/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/DataLoss/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/DeadlineExceeded/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Internal/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/OutOfRange/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/ResourceExhausted/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Unavailable/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/Unknown/",
                  "color": "#C4162A"
                },
                {
                  "alias": "/OK/",
                  "color": "#37872D"
                },
                {
                  "alias": "error",
                  "color": "#C4162A"
                }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace,  job, grpc_method, grpc_code) (rate(grpc_server_handled_total{namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{namespace}} {{job}} {{grpc_method}} {{grpc_code}}",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rate",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {
                "error": "#E24D42"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows ratio of errors compared to the total number of handled requests.",
              "fill": 10,
              "id": 14,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (namespace, job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (namespace, job) (rate(grpc_server_handled_total{namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval]))",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "error",
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Errors",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "description": "Shows how long has it taken to handle requests, in quantiles",
              "fill": 1,
              "id": 15,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "p99",
                  "color": "#FA6400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p90",
                  "color": "#E0B400",
                  "fill": 1,
                  "fillGradient": 1
                },
                {
                  "alias": "p50",
                  "color": "#37872D",
                  "fill": 10,
                  "fillGradient": 0
                }
              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.50, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p50 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.90, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p90 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                },
                {
                  "expr": "histogram_quantile(0.99, sum by (namespace, job, le) (rate(grpc_server_handling_seconds_bucket{namespace=\"$namespace\", job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "p99 {{namespace}} {{job}}",
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Duration",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "gRPC (Stream)",
          "titleSize": "h6"
        },
        {
          "collapse": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 16,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_memstats_alloc_bytes{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "alloc all {{instance}}",
                  "legendLink": null,
                  "step": 10
                },
                {
                  "expr": "go_memstats_heap_alloc_bytes{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "alloc heap {{instance}}",
                  "legendLink": null,
                  "step": 10
                },
                {
                  "expr": "rate(go_memstats_alloc_bytes_total{namespace=\"$namespace\", job=\"$job\"}[30s])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "alloc rate all {{instance}}",
                  "legendLink": null,
                  "step": 10
                },
                {
                  "expr": "rate(go_memstats_heap_alloc_bytes{namespace=\"$namespace\", job=\"$job\"}[30s])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "alloc rate heap {{instance}}",
                  "legendLink": null,
                  "step": 10
                },
                {
                  "expr": "go_memstats_stack_inuse_bytes{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "inuse heap {{instance}}",
                  "legendLink": null,
                  "step": 10
                },
                {
                  "expr": "go_memstats_heap_inuse_bytes{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "inuse stack {{instance}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory Used",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 17,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_goroutines{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{instance}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Goroutines",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            },
            {
              "aliasColors": {

              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "$datasource",
              "fill": 1,
              "id": 18,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [

              ],
              "nullPointMode": "null as zero",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [

              ],
              "spaceLength": 10,
              "span": 4,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "go_gc_duration_seconds{namespace=\"$namespace\", job=\"$job\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{quantile}} {{instance}}",
                  "legendLink": null,
                  "step": 10
                }
              ],
              "thresholds": [

              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "GC Time Quantiles",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [

                ]
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": false
                }
              ]
            }
          ],
          "repeat": null,
          "repeatIteration": null,
          "repeatRowId": null,
          "showTitle": true,
          "title": "Resources",
          "titleSize": "h6"
        }
      ],
      "schemaVersion": 14,
      "style": "dark",
      "tags": [
        "thanos-mixin",
        "observatorium"
      ],
      "templating": {
        "list": [
          {
            "current": {
              "text": "default",
              "value": "default"
            },
            "hide": 0,
            "label": null,
            "name": "datasource",
            "options": [

            ],
            "query": "prometheus",
            "refresh": 1,
            "regex": "",
            "type": "datasource"
          },
          {
            "auto": true,
            "auto_count": 300,
            "auto_min": "10s",
            "current": {
              "text": "5m",
              "value": "5m"
            },
            "hide": 0,
            "label": "interval",
            "name": "interval",
            "query": "5m,10m,30m,1h,6h,12h",
            "refresh": 2,
            "type": "interval"
          },
          {
            "allValue": null,
            "current": {

            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": false,
            "label": "namespace",
            "multi": false,
            "name": "namespace",
            "options": [

            ],
            "query": "label_values(thanos_status, namespace)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {
              "text": "all",
              "value": "$__all"
            },
            "datasource": "$datasource",
            "hide": 0,
            "includeAll": true,
            "label": "job",
            "multi": false,
            "name": "job",
            "options": [

            ],
            "query": "label_values(up{namespace=\"$namespace\", job=~\"observatorium-thanos-rule.*\"}, job)",
            "refresh": 1,
            "regex": "",
            "sort": 2,
            "tagValuesQuery": "",
            "tags": [

            ],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "auto": true,
            "auto_count": 300,
            "auto_min": "10s",
            "current": {
              "text": "5m",
              "value": "5m"
            },
            "hide": 0,
            "label": "interval",
            "name": "interval",
            "query": "5m,10m,30m,1h,6h,12h",
            "refresh": 2,
            "type": "interval"
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "UTC",
      "title": "Thanos / Rule",
      "uid": "35da848f5f92b2dc612e0c3a0577b8a1",
      "version": 0
    }
kind: ConfigMap
metadata:
  annotations:
    grafana-folder: /grafana-dashboard-definitions/Observatorium
  labels:
    grafana_dashboard: "true"
  name: grafana-dashboard-observatorium-thanos-rule
