apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboards
data:
  node-metrics.json: |
    {
      "__requires": [
        {
          "type": "panel",
          "id": "graph",
          "name": "Graph",
          "version": ""
        },
        {
          "type": "grafana",
          "id": "grafana",
          "name": "Grafana",
          "version": "3.1.0"
        },
        {
          "type": "datasource",
          "id": "prometheus",
          "name": "Prometheus",
          "version": "1.0.0"
        }
      ],
      "id": null,
      "title": "Node Metrics",
      "description": "Dashboard to view Node metrics",
      "tags": [
        "nodes",
        "node-metrics"
      ],
      "style": "dark",
      "timezone": "browser",
      "editable": true,
      "hideControls": false,
      "sharedCrosshair": false,
      "rows": [
        {
          "collapse": false,
          "editable": true,
          "height": "25px",
          "panels": [
            {
              "cacheTimeout": null,
              "colorBackground": false,
              "colorValue": false,
              "colors": [
                "rgba(245, 54, 54, 0.9)",
                "rgba(237, 129, 40, 0.89)",
                "rgba(50, 172, 45, 0.97)"
              ],
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "format": "none",
              "gauge": {
                "maxValue": 100,
                "minValue": 0,
                "show": false,
                "thresholdLabels": false,
                "thresholdMarkers": true
              },
              "id": 20,
              "interval": null,
              "links": [],
              "maxDataPoints": 100,
              "minSpan": 2,
              "nullPointMode": "connected",
              "nullText": null,
              "postfix": "",
              "postfixFontSize": "50%",
              "prefix": "",
              "prefixFontSize": "50%",
              "repeat": "node",
              "span": 12,
              "sparkline": {
                "fillColor": "rgba(31, 118, 189, 0.18)",
                "full": false,
                "lineColor": "rgb(31, 120, 193)",
                "show": false
              },
              "targets": [
                {
                  "expr": "count(node_cpu_seconds_total{namespace=\"$namespace\",node=\"$node\", mode=\"system\"}) or count(node_cpu{namespace=\"$namespace\",node=\"$node\", mode=\"system\"})",
                  "instant": true,
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "",
                  "metric": "",
                  "refId": "A",
                  "step": 14400,
                  "target": ""
                }
              ],
              "thresholds": "",
              "title": "CPU Cores",
              "type": "singlestat",
              "valueFontSize": "80%",
              "valueMaps": [
                {
                  "op": "=",
                  "text": "N/A",
                  "value": "null"
                }
              ],
              "valueName": "avg"
            }
          ],
          "title": "New row"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "decimals": 3,
              "editable": true,
              "error": false,
              "fill": 10,
              "description": "(user: The time spent in userland) -- (system: The time spent in the kernel) -- (iowait: Time spent waiting for I/O) -- (idle: Time the CPU had nothing to do -- (irq&softirq: Time servicing interrupts) -- (guest: If you are running VMs, the CPU they use) -- (steal: If you are a VM, time other VMs \"stole\" from your CPUs)",
              "grid": {
                "threshold1": 0,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)",
                "thresholdLine": false
              },
              "id": 7,
              "legend": {
                "alignAsTable": false,
                "avg": false,
                "current": false,
                "hideEmpty": false,
                "max": false,
                "min": false,
                "rightSide": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 0,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": true,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"system\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{mode}}",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"user\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "user",
                  "refId": "B",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"nice\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "nice",
                  "refId": "C",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"iowait\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "iowait",
                  "refId": "E",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"steal\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "intervalFactor": 2,
                  "legendFormat": "steal",
                  "refId": "H",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"idle\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "idle",
                  "refId": "D",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"irq\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "irq",
                  "refId": "F",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"softirq\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "softirq",
                  "refId": "G",
                  "step": 1200
                },
                {
                  "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",namespace=\"$namespace\",node=\"$node\"}[5m])) or sum(irate(node_cpu{mode=\"guest\",namespace=\"$namespace\",node=\"$node\"}[5m]))",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "guest",
                  "refId": "I",
                  "step": 1200
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "CPU",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": "%",
                  "logBase": 1,
                  "max": 100,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "CPU"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {
                "Slab": "#E5A8E2",
                "Swap": "#E24D42"
              },
              "bars": false,
              "datasource": "prometheus-meta",
              "decimals": 2,
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 17,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [
                {
                  "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/",
                  "fill": 5,
                  "stack": true
                },
                {
                  "alias": "Swap",
                  "fill": 5,
                  "stack": true
                }
              ],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "( node_memory_MemTotal_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_MemFree_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Buffers_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Cached_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapCached_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_Slab_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_PageTables_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_VmallocUsed_bytes{namespace=\"$namespace\",node=\"$node\"} ) or ( node_memory_MemTotal{namespace=\"$namespace\",node=\"$node\"} - node_memory_MemFree{namespace=\"$namespace\",node=\"$node\"} - node_memory_Buffers{namespace=\"$namespace\",node=\"$node\"} - node_memory_Cached{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapCached{namespace=\"$namespace\",node=\"$node\"} - node_memory_Slab{namespace=\"$namespace\",node=\"$node\"} - node_memory_PageTables{namespace=\"$namespace\",node=\"$node\"} - node_memory_VmallocUsed{namespace=\"$namespace\",node=\"$node\"} )",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Apps",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                },
                {
                  "expr": "node_memory_Buffers_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Buffers{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Buffers",
                  "refId": "B",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Cached_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Cached{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Cached",
                  "refId": "D",
                  "step": 1200
                },
                {
                  "expr": "node_memory_MemFree_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_MemFree{namespace=\"$namespace\",node=\"$node\"}",
                  "hide": false,
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Free",
                  "refId": "E",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Slab_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Slab{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Slab",
                  "refId": "F",
                  "step": 1200
                },
                {
                  "expr": "node_memory_SwapCached_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_SwapCached{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "SwapCached",
                  "refId": "G",
                  "step": 1200
                },
                {
                  "expr": "node_memory_PageTables_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_PageTables{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "PageTables",
                  "refId": "H",
                  "step": 1200
                },
                {
                  "expr": "node_memory_VmallocUsed_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_VmallocUsed{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "VmallocUsed",
                  "metric": "",
                  "refId": "I",
                  "step": 1200
                },
                {
                  "expr": "(node_memory_SwapTotal_bytes{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapFree{namespace=\"$namespace\",node=\"$node\"}) or (node_memory_SwapTotal{namespace=\"$namespace\",node=\"$node\"} - node_memory_SwapFree{namespace=\"$namespace\",node=\"$node\"})",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Swap",
                  "metric": "",
                  "refId": "C",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Committed_AS_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Committed_AS{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Committed",
                  "metric": "",
                  "refId": "J",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Mapped_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Mapped{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Mapped",
                  "refId": "K",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Active_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Active{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Active",
                  "metric": "",
                  "refId": "L",
                  "step": 1200
                },
                {
                  "expr": "node_memory_Inactive_bytes{namespace=\"$namespace\",node=\"$node\"} or node_memory_Inactive{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Inactive",
                  "metric": "",
                  "refId": "M",
                  "step": 1200
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": "GB",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Memory"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 13,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "node_load1{namespace=\"$namespace\",node=\"$node\"}",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "load",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Load",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Load"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "decimals": 3,
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 9,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "100.0 - 100 * (node_filesystem_avail_bytes{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{namespace=\"$namespace\",node=\"$node\",device !~'tmpfs',device!~'by-uuid'})",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{mountpoint}}",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk Space Used",
              "tooltip": {
                "msResolution": true,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "percent",
                  "logBase": 1,
                  "max": 100,
                  "min": 0,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Disk Used"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 19,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_disk_io_time_seconds_total{namespace=\"$namespace\",node=\"$node\"}[5m])/10 or irate(node_disk_io_time_ms{namespace=\"$namespace\",node=\"$node\"}[5m])/10",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{device}}",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk Utilization per Device",
              "tooltip": {
                "msResolution": false,
                "shared": false,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "percent",
                  "logBase": 1,
                  "max": 100,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Disk Utilization"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 14,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [
                {
                  "alias": "/.*_read$/",
                  "transform": "negative-Y"
                }
              ],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_disk_reads_completed_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_disk_reads_completed{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "interval": "",
                  "intervalFactor": 4,
                  "legendFormat": "{{device}}_read",
                  "metric": "",
                  "refId": "A",
                  "step": 2400,
                  "target": ""
                },
                {
                  "expr": "irate(node_disk_writes_completed_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_disk_writes_completed{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "intervalFactor": 2,
                  "legendFormat": "{{device}}_write",
                  "metric": "",
                  "refId": "B",
                  "step": 1200
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk IOs per Device",
              "tooltip": {
                "msResolution": false,
                "shared": false,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": "IO/second read (-) / write (+)",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Disk IOs per device"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 18,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [
                {
                  "alias": "/.*_read/",
                  "transform": "negative-Y"
                }
              ],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_disk_read_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512 or irate(node_disk_sectors_read{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512",
                  "interval": "",
                  "intervalFactor": 4,
                  "legendFormat": "{{device}}_read",
                  "refId": "B",
                  "step": 2400
                },
                {
                  "expr": "irate(node_disk_written_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512 or irate(node_disk_sectors_written{namespace=\"$namespace\",node=\"$node\"}[5m]) * 512",
                  "interval": "",
                  "intervalFactor": 4,
                  "legendFormat": "{{device}}_write",
                  "metric": "",
                  "refId": "A",
                  "step": 2400,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk Throughput per Device",
              "tooltip": {
                "msResolution": false,
                "shared": false,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "bytes",
                  "label": "Bytes/second read (-) / write (+)",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Disk Throughput per device"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 22,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_context_switches_total{namespace=\"$namespace\",node=\"$node\"}[5m]) or irate(node_context_switches{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "context switches",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Context Switches",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "Network Traffic"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 12,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [
                {
                  "alias": "/.*_in/",
                  "transform": "negative-Y"
                }
              ],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_network_receive_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m])*8 or irate(node_network_receive_bytes{namespace=\"$namespace\",node=\"$node\"}[5m])*8",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{device}}_in",
                  "metric": "",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                },
                {
                  "expr": "irate(node_network_transmit_bytes_total{namespace=\"$namespace\",node=\"$node\"}[5m])*8 or irate(node_network_transmit_bytes{namespace=\"$namespace\",node=\"$node\"}[5m])*8",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "{{device}}_out",
                  "refId": "B",
                  "step": 1200
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Network Traffic",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "bits",
                  "label": "bits in (-) / bits out (+)",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "title": "New row"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 21,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "node_netstat_Tcp_CurrEstab{namespace=\"$namespace\",node=\"$node\"}",
                  "intervalFactor": 2,
                  "legendFormat": "established",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Netstat",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "New row"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 23,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [
                {
                  "alias": "/.*Out.*/",
                  "transform": "negative-Y"
                },
                {
                  "alias": "Udp_NoPorts",
                  "yaxis": 2
                }
              ],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "irate(node_netstat_Udp_InDatagrams{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "intervalFactor": 2,
                  "legendFormat": "Udp_InDatagrams",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                },
                {
                  "expr": "irate(node_netstat_Udp_InErrors{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "intervalFactor": 2,
                  "legendFormat": "Udp_InErrors",
                  "refId": "B",
                  "step": 1200
                },
                {
                  "expr": "irate(node_netstat_Udp_OutDatagrams{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "interval": "",
                  "intervalFactor": 2,
                  "legendFormat": "Udp_OutDatagrams",
                  "refId": "C",
                  "step": 1200
                },
                {
                  "expr": "irate(node_netstat_Udp_NoPorts{namespace=\"$namespace\",node=\"$node\"}[5m])",
                  "intervalFactor": 2,
                  "legendFormat": "Udp_NoPorts",
                  "refId": "D",
                  "step": 1200
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "UDP Stats",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "title": "New row"
        },
        {
          "collapse": false,
          "editable": true,
          "height": "250px",
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "datasource": "prometheus-meta",
              "editable": true,
              "error": false,
              "fill": 1,
              "grid": {
                "threshold1": null,
                "threshold1Color": "rgba(216, 200, 27, 0.27)",
                "threshold2": null,
                "threshold2Color": "rgba(234, 112, 112, 0.22)"
              },
              "id": 24,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 2,
              "links": [],
              "minSpan": 2,
              "nullPointMode": "connected",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "repeat": "node",
              "seriesOverrides": [],
              "span": 12,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "node_nf_conntrack_entries_limit{namespace=\"$namespace\",node=\"$node\"} - node_nf_conntrack_entries{namespace=\"$namespace\",node=\"$node\"}",
                  "intervalFactor": 2,
                  "legendFormat": "free",
                  "refId": "A",
                  "step": 1200,
                  "target": ""
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Conntrack",
              "tooltip": {
                "msResolution": false,
                "shared": true,
                "value_type": "cumulative"
              },
              "type": "graph",
              "xaxis": {
                "show": true
              },
              "yaxes": [
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ]
            }
          ],
          "repeat": null,
          "title": "New row"
        }
      ],
      "templating": {
        "list": [
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 0,
            "includeAll": false,
            "label": null,
            "multi": false,
            "name": "namespace",
            "options": [],
            "query": "label_values(namespace)",
            "refresh": 2,
            "regex": "default|prombench-\\d+",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 0,
            "includeAll": false,
            "label": null,
            "multi": false,
            "name": "node",
            "options": [],
            "query": "label_values({namespace=\"$namespace\"}, node)",
            "refresh": 2,
            "regex": ".*",
            "sort": 0,
            "tagValuesQuery": null,
            "tags": [],
            "tagsQuery": null,
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-30m",
        "to": "now"
      },
      "timepicker": {
        "now": true,
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "annotations": {
        "list": []
      },
      "schemaVersion": 12,
      "version": 30,
      "links": [],
      "gnetId": 405
    }
  prombench.json: |
    {
      "__requires": [
        {
          "type": "grafana",
          "id": "grafana",
          "name": "Grafana",
          "version": "5.0.0"
        },
        {
          "type": "datasource",
          "id": "prometheus",
          "name": "Prometheus",
          "version": "5.0.0"
        }
      ],
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": "-- Grafana --",
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.3.x.",
      "tags": [
        "benchmark",
        "node-metrics"
      ],
      "editable": true,
      "gnetId": 6725,
      "graphTooltip": 1,
      "id": null,
      "iteration": 1532083926170,
      "links": [],
      "panels": [
      {
          "collapsed": true,
          "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 0
          },
          "id": 47,
          "panels": [
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 0,
              "y": 1
              },
              "id": 51,
              "legend": {
              "alignAsTable": true,
              "avg": true,
              "current": true,
              "max": true,
              "min": true,
              "show": true,
              "total": false,
              "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "hide": false,
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Total memory",
                  "refId": "E"
              },
              {
                  "expr": "node_memory_MemTotal_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"} - node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Used",
                  "refId": "A"
              },
              {
                  "expr": "node_memory_Buffers_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Buffers",
                  "refId": "B"
              },
              {
                  "expr": "node_memory_Cached_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Cached",
                  "refId": "C"
              },
              {
                  "expr": "node_memory_MemFree_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Free",
                  "refId": "D"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Memory",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "bytes",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          },
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 12,
              "y": 1
              },
              "id": 53,
              "legend": {
              "avg": false,
              "current": false,
              "max": false,
              "min": false,
              "show": true,
              "total": false,
              "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "irate(node_disk_io_time_seconds_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'^(md\\\\\\\\d+$|dm-)'}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - {{device}}",
                  "refId": "A"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk I/O Utilisation",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          },
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 0,
              "y": 10
              },
              "id": 57,
              "legend": {
              "avg": false,
              "current": false,
              "max": false,
              "min": false,
              "show": true,
              "total": false,
              "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "irate(node_context_switches_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Context Switches",
                  "refId": "A"
              },
              {
                  "expr": "irate(node_intr_total{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}[5m])",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Interrupts",
                  "refId": "B"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Context Switches / Interrupts",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          },
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "description": "Disk space used of all filesystems mounted",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 12,
              "y": 10
              },
              "id": 61,
              "legend": {
              "alignAsTable": true,
              "avg": true,
              "current": true,
              "max": true,
              "min": true,
              "show": true,
              "total": false,
              "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "100 - ((node_filesystem_avail_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{node=~'(test+).*',device!~'rootfs'})",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - {{mountpoint}}",
                  "refId": "A"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Disk Space Used Basic",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "transparent": false,
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "percent",
                  "label": null,
                  "logBase": 1,
                  "max": "100",
                  "min": "0",
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          },
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 0,
              "y": 19
              },
              "id": 59,
              "legend": {
              "alignAsTable": true,
              "avg": true,
              "current": true,
              "max": true,
              "min": true,
              "show": true,
              "total": false,
              "values": true
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "node_load1{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Load 1m",
                  "refId": "A"
              },
              {
                  "expr": "node_load5{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Load 5m",
                  "refId": "B"
              },
              {
                  "expr": "node_load15{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - Load 15m",
                  "refId": "C"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "System Load",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          },
          {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus-meta",
              "fill": 1,
              "gridPos": {
              "h": 9,
              "w": 12,
              "x": 12,
              "y": 19
              },
              "id": 55,
              "legend": {
              "avg": false,
              "current": false,
              "max": false,
              "min": false,
              "show": true,
              "total": false,
              "values": false
              },
              "lines": true,
              "linewidth": 1,
              "links": [],
              "nullPointMode": "null",
              "percentage": false,
              "pointradius": 5,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
              {
                  "expr": "1 - node_filesystem_free_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\",fstype!='rootfs',mountpoint!~'/(run|var).*',mountpoint!=''} / node_filesystem_size_bytes{job=\"node-exporter\",namespace=\"prombench-[[pr-number]]\",node=~\"(test+).*\"}",
                  "format": "time_series",
                  "intervalFactor": 1,
                  "legendFormat": "{{node}} - {{mountpoint}}",
                  "refId": "A"
              }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeShift": null,
              "title": "Filesystem Fullness",
              "tooltip": {
              "shared": true,
              "sort": 0,
              "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
              "buckets": null,
              "mode": "time",
              "name": null,
              "show": true,
              "values": []
              },
              "yaxes": [
              {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              },
              {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
              }
              ]
          }
          ],
          "title": "Node Metrics",
          "type": "row"
      },
      {
          "collapsed": false,
          "gridPos": {
          "h": 1,
          "w": 24,
          "x": 0,
          "y": 1
          },
          "id": 45,
          "panels": [],
          "title": "Prometheus Benchmark",
          "type": "row"
      },
      {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 2
          },
          "id": 40,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_build_info{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{prometheus}} - {{version}} - {{revision}}",
              "refId": "A"
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Prometheus Version",
          "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 2
          },
          "id": 42,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "time() - process_start_time_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{prometheus}} - Uptime",
              "refId": "A"
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Uptime",
          "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "dtdurations",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Max": "#e24d42",
          "Open": "#508642"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 2
          },
          "id": 41,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "process_max_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{prometheus}} - Max",
              "refId": "A"
          },
          {
              "expr": "process_open_fds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{prometheus}} - Open",
              "refId": "B"
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "File Descriptors",
          "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "Time series": "#70dbed"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 9
          },
          "id": 3,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_tsdb_head_series{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Time series",
              "metric": "prometheus_local_storage_memory_series",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Time series",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 9
          },
          "id": 26,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Head Appenders",
              "metric": "prometheus_local_storage_memory_series",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Active Appenders",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "samples/s": "#e5a8e2"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 9
          },
          "id": 1,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - samples/s",
              "metric": "prometheus_local_storage_ingested_samples_total",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Samples Appended/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "To persist": "#9AC48A"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 16
          },
          "id": 2,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
          {
              "alias": "/Max.*/",
              "fill": 0
          }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Chunks",
              "metric": "prometheus_local_storage_memory_chunks",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Chunks",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 16
          },
          "id": 4,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Created",
              "metric": "prometheus_local_storage_chunk_ops_total",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Chunks Created",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "ops",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "Removed": "#e5ac0e"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 16
          },
          "id": 25,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Removed",
              "metric": "prometheus_local_storage_chunk_ops_total",
              "refId": "B",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Chunks Removed",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "ops",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max": "#447ebc",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "Min": "#447ebc",
          "Now": "#7eb26d"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 23
          },
          "id": 28,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
          {
              "alias": "Max",
              "fillBelowTo": "Min",
              "lines": false
          }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Min",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          },
          {
              "expr": "time() * 1000",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "Now",
              "refId": "C"
          },
          {
              "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Max",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Head Time Range",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "decimals": null,
              "format": "dateTimeAsIso",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 23
          },
          "id": 29,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_head_gc_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - GC Time/s",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg Head GC Time/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 23
          },
          "id": 14,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
          {
              "alias": "Queue length",
              "yaxis": 2
          }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Blocks Loaded",
              "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Blocks Loaded",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Allocated bytes": "#F9BA8F",
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "RSS": "#890F02"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 30
          },
          "id": 7,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Allocated Bytes/s",
              "metric": "go_memstats_alloc_bytes",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Allocations",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Failed Compactions": "#bf1b00",
          "Failed Reloads": "#bf1b00",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 30
          },
          "id": 30,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Reloads",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "TSDB Reloads/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Failed Compactions": "#bf1b00",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}": "#bf1b00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 30
          },
          "id": 32,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - WAL Corruptions",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          },
          {
              "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Reload Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
          },
          {
              "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Head Series Not Found",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "C",
              "step": 10
          },
          {
              "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Compaction Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "D",
              "step": 10
          },
          {
              "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Retention Cutoff Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "E",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "TSDB Problems/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Allocated bytes": "#F9BA8F",
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "RSS": "#890F02"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 37
          },
          "id": 8,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_retention_cutoffs_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Retention Cutoffs",
              "metric": "last",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Retention Cutoffs/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Failed Compactions": "#bf1b00",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 37
          },
          "id": 31,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Fsync Latency",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "WAL Fsync Latency",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Failed Compactions": "#bf1b00",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 37
          },
          "id": 65,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Truncate Latency",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "WAL Truncate Latency",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 44
          },
          "id": 35,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_compaction_chunk_size_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Bytes/Sample",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "First Compaction, Avg Bytes/Sample",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "decimals": null,
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 44
          },
          "id": 27,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_compaction_chunk_range_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) or rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Chunk Time Range",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "First Compaction, Avg Chunk Time Range",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "decimals": null,
              "format": "ms",
              "label": "",
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 44
          },
          "id": 34,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Chunk Samples",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "First Compaction, Avg Chunk Samples",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Failed Compactions": "#bf1b00",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 51
          },
          "id": 19,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Compactions",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Compactions/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 51
          },
          "id": 33,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m]) / rate(prometheus_tsdb_compaction_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[10m])",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}}",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg Compaction Time/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "decimals": 2,
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 51
          },
          "id": 9,
          "legend": {
          "alignAsTable": false,
          "avg": false,
          "current": false,
          "hideEmpty": false,
          "max": false,
          "min": false,
          "rightSide": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Irate",
              "metric": "prometheus_local_storage_ingested_samples_total",
              "refId": "A",
              "step": 10
          },
          {
              "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[5m])",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - 5m rate",
              "metric": "prometheus_local_storage_ingested_samples_total",
              "refId": "B",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "CPU",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [
              "avg"
          ]
          },
          "yaxes": [
          {
              "format": "none",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Allocated bytes": "#7EB26D",
          "Allocated bytes - 1m max": "#BF1B00",
          "Allocated bytes - 1m min": "#BF1B00",
          "Allocated bytes - 5m max": "#BF1B00",
          "Allocated bytes - 5m min": "#BF1B00",
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833",
          "RSS": "#447EBC"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "decimals": null,
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 58
          },
          "id": 6,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
          {
              "alias": "/-/",
              "fill": 0
          }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "process_resident_memory_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - RSS",
              "metric": "process_resident_memory_bytes",
              "refId": "B",
              "step": 10
          },
          {
              "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Target heap size",
              "metric": "go_memstats_alloc_bytes",
              "refId": "D",
              "step": 10
          },
          {
              "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Next GC",
              "metric": "go_memstats_next_gc_bytes",
              "refId": "C",
              "step": 10
          },
          {
              "expr": "go_memstats_alloc_bytes{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Allocated",
              "metric": "go_memstats_alloc_bytes",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Memory",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "",
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 58
          },
          "id": 38,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - {{handler}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "HTTP requests/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "",
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 58
          },
          "id": 37,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - {{handler}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg HTTP request latency",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "Time spent in prepare_time mode, per second",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 65
          },
          "id": 64,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"prepare_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg query engine timings/s - prepare_time",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "Average Time spent in inner_eval mode, per second",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 65
          },
          "id": 24,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"inner_eval\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg query engine timings/s - inner_eval",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "Time spent in queue_time mode, per second",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 65
          },
          "id": 63,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"queue_time\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg query engine timings/s - queue_time",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "description": "Average Time spent in result_sort mode, per second",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 72
          },
          "id": 62,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m]) / rate(prometheus_engine_query_duration_seconds_count{job=\"prometheus\",slice=\"result_sort\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Avg query engine timings/s - result_sort",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 8,
          "y": 72
          },
          "id": 22,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])  ",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Rule group missed",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "B",
              "step": 10
          },
          {
              "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Rule evals failed",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "C",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Rule group evaluation problems/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 1,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 72
          },
          "id": 23,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}[1m])",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Rule evaluation duration",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Evaluation time of rule groups/s",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
          "aliasColors": {
          "Chunks": "#1F78C1",
          "Chunks to persist": "#508642",
          "Interval": "#890f02",
          "Last Duration": "#f9934e",
          "Max chunks": "#052B51",
          "Max to persist": "#3F6833"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus-meta",
          "editable": true,
          "error": false,
          "fill": 0,
          "gridPos": {
          "h": 7,
          "w": 8,
          "x": 0,
          "y": 79
          },
          "id": 43,
          "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": "RuleGroup",
          "repeatDirection": "h",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
          {
              "expr": "prometheus_rule_group_interval_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Interval",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
          },
          {
              "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\",rule_group=~\"$RuleGroup\"}\n",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{prometheus}} - Last Duration",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "B",
              "step": 10
          }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Rule Group: $RuleGroup",
          "tooltip": {
          "msResolution": false,
          "shared": true,
          "sort": 0,
          "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
          },
          "yaxes": [
          {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": "0",
              "show": true
          },
          {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
          }
          ]
      },
      {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 79
      },
      "id": 67,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "prometheus_tsdb_isolation_low_watermark{namespace='prombench-[[pr-number]]'}",
          "legendFormat": "{{prometheus}} - low watermark",
          "refId": "A"
        },
        {
          "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-[[pr-number]]'}",
          "interval": "",
          "legendFormat": "{{prometheus}} - high watermark",
          "refId": "B"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "TSDB Isolation Watermarks",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
        "aliasColors": {},
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "fill": 0,
        "fillGradient": 0,
        "gridPos": {
          "h": 7,
          "w": 8,
          "x": 16,
          "y": 79
        },
        "id": 68,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "dataLinks": []
        },
        "percentage": false,
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
            "expr": "prometheus_tsdb_isolation_high_watermark{namespace='prombench-[[pr-number]]'} - prometheus_tsdb_isolation_low_watermark{namespace='prombench-[[pr-number]]'}",
            "legendFormat": "{{prometheus}}",
            "refId": "A"
          }
        ],
        "thresholds": [],
        "timeFrom": null,
        "timeRegions": [],
        "timeShift": null,
        "title": "TSDB Isolation Watermarks Difference",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": []
        },
        "yaxes": [
          {
            "format": "short",
            "label": null,
            "logBase": 1,
            "max": null,
            "min": null,
            "show": true
          },
          {
            "format": "short",
            "label": null,
            "logBase": 1,
            "max": null,
            "min": null,
            "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
      }
      ],
      "refresh": false,
      "schemaVersion": 16,
      "style": "dark",
      "templating": {
        "list": [
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 2,
            "includeAll": true,
            "label": null,
            "multi": false,
            "name": "RuleGroup",
            "options": [],
            "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",namespace=\"prombench-[[pr-number]]\",prometheus=~\"(test+).*\"}",
            "refresh": 2,
            "regex": ".*rule_group=\"(.*?)\".*",
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          },
          {
            "allValue": null,
            "current": {},
            "datasource": "prometheus-meta",
            "hide": 0,
            "includeAll": false,
            "label": null,
            "multi": false,
            "name": "pr-number",
            "options": [],
            "query": "label_values(namespace)",
            "refresh": 2,
            "regex": "prombench-(\\d+)",
            "sort": 0,
            "tagValuesQuery": null,
            "tags": [],
            "tagsQuery": null,
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-30m",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "5s",
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ],
        "time_options": [
          "5m",
          "15m",
          "1h",
          "6h",
          "12h",
          "24h",
          "2d",
          "7d",
          "30d"
        ]
      },
      "timezone": "browser",
      "title": "Prombench",
      "uid": "7gmLoNDmz",
      "version": 3
    }
