apiVersion: v1
data:
  prometheus-dashboard.yaml: |
    apiVersion: 1

    providers:
    - name: 'default'
      orgId: 1
      folder: ''
      type: file
      disableDeletion: false
      updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
      options:
        path: /opt/grafana-provision/dashboards
  prombench.json: |
    {
      "__requires": [
        {
          "type": "grafana",
          "id": "grafana",
          "name": "Grafana",
          "version": "5.0.0"
        },
        {
          "type": "datasource",
          "id": "prometheus",
          "name": "Prometheus",
          "version": "5.0.0"
        }
      ],
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": "-- Grafana --",
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.3.x.",
      "editable": true,
      "gnetId": 6725,
      "graphTooltip": 1,
      "id": null,
      "iteration": 1532083926170,
      "links": [],
      "panels": [
        {
          "collapsed": true,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 0
          },
          "id": 47,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 0,
                "y": 1
              },
              "id": 51,
              "legend": {
                "alignAsTable": true,
                "avg": true,
                "current": true,
                "max": true,
                "min": true,
                "show": true,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "hide": false,
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Total memory",
                  "refId": "E"
                },
                {
                  "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Used",
                  "refId": "A"
                },
                {
                  "expr": "node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Buffers",
                  "refId": "B"
                },
                {
                  "expr": "node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Cached",
                  "refId": "C"
                },
                {
                  "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Free",
                  "refId": "D"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 12,
                "y": 1
              },
              "id": 53,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_disk_io_time_seconds_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'^(md\\\\\\\\d+$|dm-)'}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - {{"{{"}}device{{"}}"}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk I/O Utilisation",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 0,
                "y": 10
              },
              "id": 57,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_context_switches_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Context Switches",
                  "refId": "A"
                },
                {
                  "expr": "irate(node_intr_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Interrupts",
                  "refId": "B"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Context Switches / Interrupts",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "Disk space used of all filesystems mounted",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 12,
                "y": 10
              },
              "id": 61,
              "legend": {
                "alignAsTable": true,
                "avg": true,
                "current": true,
                "max": true,
                "min": true,
                "show": true,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{node=~'(test+).*',device!~'rootfs'})",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - {{"{{"}}mountpoint{{"}}"}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk Space Used Basic",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "transparent": false,
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "percent",
                  "label": null,
                  "logBase": 1,
                  "max": "100",
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 0,
                "y": 19
              },
              "id": 59,
              "legend": {
                "alignAsTable": true,
                "avg": true,
                "current": true,
                "max": true,
                "min": true,
                "show": true,
                "total": false,
                "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "node_load1{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Load 1m",
                  "refId": "A"
                },
                {
                  "expr": "node_load5{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Load 5m",
                  "refId": "B"
                },
                {
                  "expr": "node_load15{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - Load 15m",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "System Load",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 12,
                "y": 19
              },
              "id": 55,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "1 - node_filesystem_free_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}node{{"}}"}} - {{"{{"}}mountpoint{{"}}"}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Filesystem Fullness",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Node Metrics",
          "type": "row"
        },
        {
          "collapsed": true,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 1
          },
          "id": 45,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 2
              },
              "id": 40,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_build_info{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - {{"{{"}}version{{"}}"}} - {{"{{"}}revision{{"}}"}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Prometheus Version",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 2
              },
              "id": 42,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "time() - process_start_time_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Uptime",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Uptime",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "dtdurations",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Max": "#e24d42",
                "Open": "#508642"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 2
              },
              "id": 41,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "process_max_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Max",
                  "refId": "A"
                },
                {
                  "expr": "process_open_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Open",
                  "refId": "B"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "File Descriptors",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "Time series": "#70dbed"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 9
              },
              "id": 3,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_tsdb_head_series{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Time series",
                  "metric": "prometheus_local_storage_memory_series",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Time series",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 9
              },
              "id": 26,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Head Appenders",
                  "metric": "prometheus_local_storage_memory_series",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Active Appenders",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "samples/s": "#e5a8e2"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 9
              },
              "id": 1,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - samples/s",
                  "metric": "prometheus_local_storage_ingested_samples_total",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Samples Appended/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": "",
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "To persist": "#9AC48A"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 16
              },
              "id": 2,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "/Max.*/",
                  "fill": 0
                }
              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Chunks",
                  "metric": "prometheus_local_storage_memory_chunks",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Chunks",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 16
              },
              "id": 4,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Created",
                  "metric": "prometheus_local_storage_chunk_ops_total",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Chunks Created",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "Removed": "#e5ac0e"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 16
              },
              "id": 25,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Removed",
                  "metric": "prometheus_local_storage_chunk_ops_total",
                  "refId": "B",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Chunks Removed",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "ops",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max": "#447ebc",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "Min": "#447ebc",
                "Now": "#7eb26d"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 23
              },
              "id": 28,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "Max",
                  "fillBelowTo": "Min",
                  "lines": false
                }
              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Min",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                },
                {
                  "expr": "time() * 1000",
                  "format": "time_series",
                  "hide": false,
                  "intervalFactor": 2,
                  "legendFormat": "Now",
                  "refId": "C"
                },
                {
                  "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Max",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "B",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head Time Range",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "dateTimeAsIso",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 23
              },
              "id": 29,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - GC Time/s",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Head GC Time/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 23
              },
              "id": 14,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "Queue length",
                  "yaxis": 2
                }
              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Blocks Loaded",
                  "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Blocks Loaded",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Failed Compactions": "#bf1b00",
                "Failed Reloads": "#bf1b00",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 30
              },
              "id": 30,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Reloads",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "TSDB Reloads/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Failed Compactions": "#bf1b00",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 30
              },
              "id": 31,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Fsync Latency",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Truncate Latency",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "B",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "WAL Fsync&Truncate Latency",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Failed Compactions": "#bf1b00",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 30
              },
              "id": 32,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - WAL Corruptions",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Reload Failures",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "B",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Head Series Not Found",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "C",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Compaction Failures",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "D",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Retention Cutoff Failures",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "E",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "TSDB Problems/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Failed Compactions": "#bf1b00",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 37
              },
              "id": 19,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Compactions",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Compactions/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 37
              },
              "id": 33,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}}",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Compaction Time/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Allocated bytes": "#F9BA8F",
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "RSS": "#890F02"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 37
              },
              "id": 8,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_retention_cutoffs_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Retention Cutoffs",
                  "metric": "last",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Retention Cutoffs/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 44
              },
              "id": 27,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Chunk Time Range",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "First Compaction, Avg Chunk Time Range",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "ms",
                  "label": "",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 44
              },
              "id": 35,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Bytes/Sample",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "First Compaction, Avg Bytes/Sample",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 44
              },
              "id": 34,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Chunk Samples",
                  "metric": "prometheus_local_storage_series_chunks_persisted_count",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "First Compaction, Avg Chunk Samples",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Allocated bytes": "#7EB26D",
                "Allocated bytes - 1m max": "#BF1B00",
                "Allocated bytes - 1m min": "#BF1B00",
                "Allocated bytes - 5m max": "#BF1B00",
                "Allocated bytes - 5m min": "#BF1B00",
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "RSS": "#447EBC"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "decimals": null,
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 51
              },
              "id": 6,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "/-/",
                  "fill": 0
                }
              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "process_resident_memory_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - RSS",
                  "metric": "process_resident_memory_bytes",
                  "refId": "B",
                  "step": 10
                },
                {
                  "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Target heap size",
                  "metric": "go_memstats_alloc_bytes",
                  "refId": "D",
                  "step": 10
                },
                {
                  "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Next GC",
                  "metric": "go_memstats_next_gc_bytes",
                  "refId": "C",
                  "step": 10
                },
                {
                  "expr": "go_memstats_alloc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Allocated",
                  "metric": "go_memstats_alloc_bytes",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Allocated bytes": "#F9BA8F",
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833",
                "RSS": "#890F02"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 51
              },
              "id": 7,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Allocated Bytes/s",
                  "metric": "go_memstats_alloc_bytes",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Allocations",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "decimals": 2,
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 51
              },
              "id": 9,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "hideEmpty": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Irate",
                  "metric": "prometheus_local_storage_ingested_samples_total",
                  "refId": "A",
                  "step": 10
                },
                {
                  "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[5m])",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - 5m rate",
                  "metric": "prometheus_local_storage_ingested_samples_total",
                  "refId": "B",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "CPU",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": [
                  "avg"
                ]
              },
              "yaxes": [
                {
                  "format": "none",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "",
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 58
              },
              "id": 38,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - {{"{{"}}handler{{"}}"}}",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "HTTP requests/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "",
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 58
              },
              "id": 37,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - {{"{{"}}handler{{"}}"}}",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "HTTP request latency",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 58
              },
              "id": 36,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - {{"{{"}}handler{{"}}"}}",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Time spent in HTTP requests/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "Time spent in each mode, per second",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 0,
                "y": 65
              },
              "id": 24,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - {{"{{"}}slice{{"}}"}}",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Query engine timings/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 8,
                "y": 65
              },
              "id": 22,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])  ",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Rule group missed",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "B",
                  "step": 10
                },
                {
                  "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Rule evals failed",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "C",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rule group evaulation problems/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "gridPos": {
                "h": 7,
                "w": 8,
                "x": 16,
                "y": 65
              },
              "id": 23,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Rule evaluation duration",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Evaluation time of rule groups/s",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            },
            {
              "aliasColors": {
                "Chunks": "#1F78C1",
                "Chunks to persist": "#508642",
                "Interval": "#890f02",
                "Last Duration": "#f9934e",
                "Max chunks": "#052B51",
                "Max to persist": "#3F6833"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 0,
              "gridPos": {
                "h": 7,
                "w": 12,
                "x": 0,
                "y": 72
              },
              "id": 43,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "RuleGroup",
              "repeatDirection": "h",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Interval",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "A",
                  "step": 10
                },
                {
                  "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n",
                  "format": "time_series",
                  "intervalFactor": 2,
                  "legendFormat": "{{"{{"}}prometheus{{"}}"}} - Last Duration",
                  "metric": "prometheus_local_storage_memory_chunkdescs",
                  "refId": "B",
                  "step": 10
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Rule Group: $RuleGroup",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "s",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Prometheus Benchmark",
          "type": "row"
        }
      ],
      "refresh": false,
      "schemaVersion": 16,
      "style": "dark",
      "tags": [],
      "templating": {
        "list": [
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 2,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "RuleGroup",
            "options": [],
            "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
            "refresh": 2,
            "regex": ".*rule_group=\"(.*?)\".*",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 0,
            "includeAll": false,
            "label": null,
            "multi": false,
            "name": "pr-number",
            "options": [],
            "query": "label_values(namespace)",
            "refresh": 2,
            "regex": "prombench-(\\d+)",
            "sort": 0,
            "tagValuesQuery": null,
            "tags": [],
            "tagsQuery": null,
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-30m",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "browser",
      "title": "Prombench",
      "uid": "7gmLoNDmz",
      "version": 3
    }
kind: ConfigMap
metadata:
  name: grafana-dashboards
