apiVersion: v1
kind: ConfigMap
metadata:
  name: tempo-operational
data:
  tempo-operational.json: |-
    {
    "annotations": {
      "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      },
      {
        "datasource": "$logsds",
        "enable": true,
        "expr": "{cluster=\"$cluster\", diff_namespace=\"$namespace\", container=\"kube-diff-logger\"}",
        "hide": true,
        "iconColor": "rgba(255, 96, 96, 1)",
        "name": "diffs",
        "showIn": 0,
        "target": {

        }
      }
      ]
    },
    "editable": true,
    "gnetId": null,
    "graphTooltip": 1,
    "iteration": 1609831418010,
    "links": [

    ],
    "panels": [
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 38,
      "panels": [

      ],
      "title": "General",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 24,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "gcs",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 3,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 25,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "go heap",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "decbytes",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 6,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 23,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
        "legendFormat": "{{job}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "goroutines",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 9,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 42,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])",
        "interval": "",
        "intervalFactor": 5,
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "cpu",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 12,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 43,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "working set",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "decbytes",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 15,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 44,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
        "hide": false,
        "interval": "",
        "legendFormat": "rx-{{pod}}",
        "refId": "A"
        },
        {
        "expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
        "hide": false,
        "interval": "",
        "legendFormat": "tx-{{pod}}",
        "refId": "B"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "tx/rx",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "decbytes",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 18,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 45,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}",
        "legendFormat": "{{persistentvolumeclaim}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "data volume free",
      "tooltip": {
        "shared": true,
        "sort": 1,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "decbytes",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 21,
        "y": 1
      },
      "hiddenSeries": false,
      "id": 46,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])",
        "interval": "",
        "legendFormat": "{{exported_pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "bad words",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 49,
      "panels": [

      ],
      "title": "Maintenance",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 33,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "tempodb_work_queue_length{job=~\"$namespace/$component\"} / tempodb_work_queue_max{job=~\"$namespace/$component\"}",
        "legendFormat": "{{instance}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "%age total work queue",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "decimals": null,
        "format": "percentunit",
        "label": "",
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 3,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 32,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(increase(tempodb_compaction_errors_total{job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
        "legendFormat": "compaction_err",
        "refId": "B"
        },
        {
        "expr": "sum(increase(tempodb_retention_errors_total{job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
        "legendFormat": "retention_err",
        "refId": "C"
        },
        {
        "expr": "sum(increase(tempodb_blocklist_poll_errors_total{job=~\"$namespace/$component\"}[$__rate_interval])) by (job)",
        "legendFormat": "blocklist_err",
        "refId": "D"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "maintenance errors",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 6,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 35,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": false,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Blocklist Poll Duration",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 9,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 47,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "avg(tempodb_blocklist_length{job=~\"$namespace/$component\"}) by (tenant)",
        "instant": false,
        "interval": "",
        "legendFormat": "{{tenant}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Blocklist Length",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 12,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 51,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": false,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "retention duration",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 15,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 53,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(increase(tempodb_retention_deleted_total{job=~\"$namespace/$component\"}[$__rate_interval]))",
        "interval": "",
        "legendFormat": "deleted",
        "refId": "A"
        },
        {
        "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{job=~\"$namespace/$component\"}[$__rate_interval]))",
        "interval": "",
        "legendFormat": "marked_for_deletion",
        "refId": "B"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "retention",
      "tooltip": {
        "shared": true,
        "sort": 2,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 18,
        "y": 7
      },
      "hiddenSeries": false,
      "id": 70,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0",
        "hide": false,
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Container Restarts",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 12
      },
      "id": 21,
      "panels": [

      ],
      "title": "Read",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 10,
        "w": 6,
        "x": 0,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 34,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
        "hide": false,
        "interval": "",
        "legendFormat": "{{status_code}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Queries/Sec (Query Frontend)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 10,
        "w": 6,
        "x": 6,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 78,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Query Latency (Gateway)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 12,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 17,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Query Latency (Query Frontend)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 16,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 89,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Query Latency (Querier)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 20,
        "y": 13
      },
      "hiddenSeries": false,
      "id": 13,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_query_bytes_read_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_query_bytes_read_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_query_bytes_read_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Query  Bytes Read",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "decbytes",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 12,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 3,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Query Latency (Ingester)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 16,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 15,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_query_reads_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (layer, le))",
        "hide": false,
        "interval": "",
        "legendFormat": "{{layer}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "p50 Reads",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 19,
        "y": 18
      },
      "hiddenSeries": false,
      "id": 14,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_query_reads_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (layer, le))",
        "hide": false,
        "interval": "",
        "legendFormat": "{{layer}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "p99 Reads",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 19,
      "panels": [

      ],
      "title": "Write",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 0,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 10,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Spans Received",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 3,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 9,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
        "expr": "rate(tempo_ingester_traces_created_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Traces Created",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 6,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 8,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(increase(tempo_ingester_blocks_flushed_total{job=\"$namespace/ingester\"}[1h]))",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Blocks Flushed (1h)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 9,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 12,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "increase(tempo_ingester_failed_flushes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0",
        "interval": "",
        "legendFormat": "{{pod}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Failed Flushes",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 12,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 26,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(increase(tempo_ingester_blocks_cleared_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Blocks Cleared (1h)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 15,
        "y": 24
      },
      "hiddenSeries": false,
      "id": 36,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": false,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": true,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Flush Duration",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 10,
        "w": 7,
        "x": 0,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 71,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempo_request_duration_seconds_count{job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)",
        "interval": "",
        "legendFormat": "{{status_code}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Pushes/sec (gateway)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": "",
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 10,
        "w": 6,
        "x": 7,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 72,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
        "interval": "",
        "legendFormat": "accepted",
        "refId": "A"
        },
        {
        "expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
        "interval": "",
        "legendFormat": "refused",
        "refId": "B"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "OTEL Distributor Spans/second",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "none",
        "label": "",
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 13,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 79,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Push Latency (Gateway)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 13,
        "y": 34
      },
      "hiddenSeries": false,
      "id": 2,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Pusher/Push\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".99",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Pusher/Push\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".9",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/ingester\", route=\"/tempopb.Pusher/Push\"}[$__rate_interval])) by (le))",
        "interval": "",
        "legendFormat": ".5",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Push Latency (Ingester)",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 39
      },
      "id": 74,
      "panels": [

      ],
      "title": "Memcached",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        },
        "links": [

        ]
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 40
      },
      "hiddenSeries": false,
      "id": 75,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(cortex_memcache_request_duration_seconds_count{job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)",
        "interval": "",
        "legendFormat": "{{status_code}}-{{method}}",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Requests/Second",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        },
        "links": [

        ]
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 40
      },
      "hiddenSeries": false,
      "id": 76,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "histogram_quantile(.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
        "interval": "",
        "legendFormat": ".99-{{method}}",
        "refId": "A"
        },
        {
        "expr": "histogram_quantile(.9, sum(rate(cortex_memcache_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
        "interval": "",
        "legendFormat": ".9-{{method}}",
        "refId": "B"
        },
        {
        "expr": "histogram_quantile(.5, sum(rate(cortex_memcache_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))",
        "interval": "",
        "legendFormat": ".5-{{method}}",
        "refId": "C"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Latency By Operation",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "s",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "collapsed": true,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 48
      },
      "id": 28,
      "panels": [
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 0,
          "y": 41
        },
        "hiddenSeries": false,
        "id": 31,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum(rate(tempodb_gcs_request_duration_seconds_count{job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, operation)",
          "interval": "",
          "legendFormat": "{{status_code}}-{{operation}}",
          "refId": "A"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "Requests/Second",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 12,
          "x": 12,
          "y": 41
        },
        "hiddenSeries": false,
        "id": 30,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(.99, sum(rate(tempodb_gcs_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
          "legendFormat": ".99-{{operation}}",
          "refId": "A"
          },
          {
          "expr": "histogram_quantile(.9, sum(rate(tempodb_gcs_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
          "legendFormat": ".9-{{operation}}",
          "refId": "B"
          },
          {
          "expr": "histogram_quantile(.5, sum(rate(tempodb_gcs_request_duration_seconds_bucket{job=~\"$namespace/$component\"}[$__rate_interval])) by (operation, le))",
          "legendFormat": ".5-{{operation}}",
          "refId": "C"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "Latency By Operation",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "s",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        }
      ],
      "title": "GCS",
      "type": "row"
      },
      {
      "collapsed": true,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 49
      },
      "id": 56,
      "panels": [
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 6,
          "x": 0,
          "y": 42
        },
        "hiddenSeries": false,
        "id": 58,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": false,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
          "interval": "",
          "legendFormat": "{{instance}}",
          "refId": "A"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "Gossip Health",
        "tooltip": {
          "shared": true,
          "sort": 2,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 6,
          "x": 6,
          "y": 42
        },
        "hiddenSeries": false,
        "id": 62,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": false,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}",
          "interval": "",
          "legendFormat": "{{instance}}",
          "refId": "A"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "Node Health",
        "tooltip": {
          "shared": true,
          "sort": 2,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 6,
          "x": 12,
          "y": 42
        },
        "hiddenSeries": false,
        "id": 63,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": false,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
          "interval": "",
          "legendFormat": "min",
          "refId": "A"
          },
          {
          "expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
          "interval": "",
          "legendFormat": "max",
          "refId": "B"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "Member Count",
        "tooltip": {
          "shared": true,
          "sort": 2,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$ds",
        "fieldConfig": {
          "defaults": {
          "custom": {

          },
          "links": [

          ]
          },
          "overrides": [

          ]
        },
        "fill": 1,
        "fillGradient": 0,
        "gridPos": {
          "h": 8,
          "w": 6,
          "x": 18,
          "y": 42
        },
        "hiddenSeries": false,
        "id": 60,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": false,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "nullPointMode": "null",
        "options": {
          "alertThreshold": true
        },
        "percentage": false,
        "pluginVersion": "7.2.1",
        "pointradius": 2,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
          "interval": "",
          "legendFormat": "min",
          "refId": "A"
          },
          {
          "expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})",
          "interval": "",
          "legendFormat": "max",
          "refId": "B"
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeRegions": [

        ],
        "timeShift": null,
        "title": "KV Store Count",
        "tooltip": {
          "shared": true,
          "sort": 2,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
          }
        ],
        "yaxis": {
          "align": false,
          "alignLevel": null
        }
        }
      ],
      "title": "Ring/Memberlist",
      "type": "row"
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 50
      },
      "id": 69,
      "panels": [

      ],
      "title": "Vulture",
      "type": "row"
      },
      {
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "color": {
          "mode": "thresholds"
        },
        "custom": {

        },
        "mappings": [

        ],
        "thresholds": {
          "mode": "absolute",
          "steps": [
          {
            "color": "green",
            "value": null
          },
          {
            "color": "red",
            "value": 0.050000000000000003
          }
          ]
        },
        "unit": "percentunit"
        },
        "overrides": [
        {
          "matcher": {
          "id": "byName",
          "options": "1800"
          },
          "properties": [
          {
            "id": "displayName",
            "value": ".5h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "3600"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "1h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "10800"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "3h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "21600"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "6h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "43200"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "12h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "86400"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "24h"
          }
          ]
        }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 51
      },
      "id": 77,
      "options": {
        "displayMode": "gradient",
        "orientation": "auto",
        "reduceOptions": {
        "calcs": [
          "mean"
        ],
        "fields": "",
        "values": false
        },
        "showUnfilled": true,
        "text": {

        }
      },
      "pluginVersion": "7.4.0-9155pre",
      "targets": [
        {
        "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"notfound\"}[1h])) by (secondsago) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h])) by (secondsago)",
        "interval": "",
        "legendFormat": "{{secondsago}}",
        "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Traces Not Found",
      "type": "bargauge"
      },
      {
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "color": {
          "mode": "thresholds"
        },
        "custom": {

        },
        "mappings": [

        ],
        "thresholds": {
          "mode": "absolute",
          "steps": [
          {
            "color": "green",
            "value": null
          },
          {
            "color": "red",
            "value": 0.050000000000000003
          }
          ]
        },
        "unit": "percentunit"
        },
        "overrides": [
        {
          "matcher": {
          "id": "byName",
          "options": "1800"
          },
          "properties": [
          {
            "id": "displayName",
            "value": ".5h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "3600"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "1h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "10800"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "3h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "21600"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "6h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "43200"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "12h"
          }
          ]
        },
        {
          "matcher": {
          "id": "byName",
          "options": "86400"
          },
          "properties": [
          {
            "id": "displayName",
            "value": "24h"
          }
          ]
        }
        ]
      },
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 51
      },
      "id": 67,
      "options": {
        "displayMode": "gradient",
        "orientation": "auto",
        "reduceOptions": {
        "calcs": [
          "mean"
        ],
        "fields": "",
        "values": false
        },
        "showUnfilled": true,
        "text": {

        }
      },
      "pluginVersion": "7.4.0-9155pre",
      "targets": [
        {
        "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"missingspans\"}[1h])) by (secondsago) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h])) by (secondsago)",
        "interval": "",
        "legendFormat": "{{secondsago}}",
        "refId": "A"
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Traces Missing Spans",
      "type": "bargauge"
      },
      {
      "collapsed": false,
      "datasource": null,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 60
      },
      "id": 81,
      "panels": [

      ],
      "title": "Compactor",
      "type": "row"
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 4,
        "x": 0,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 83,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\",job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Objects Combined / s",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "$$hashKey": "object:150",
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "$$hashKey": "object:151",
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "description": "",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 4,
        "x": 4,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 85,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempodb_compaction_objects_written{cluster=\"$cluster\",job=\"$namespace/compactor\"}[$__rate_interval])) by (level)",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Objects Written / s",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": null,
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 4,
        "x": 8,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 88,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(rate(tempodb_compaction_bytes_written{cluster=\"$cluster\",job=\"$namespace/compactor\"}[$__rate_interval])) by (level)",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Bytes Written / s",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      },
      {
      "aliasColors": {

      },
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$ds",
      "fieldConfig": {
        "defaults": {
        "custom": {

        }
        },
        "overrides": [

        ]
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 6,
        "w": 4,
        "x": 12,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 86,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": false,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.4.0-9155pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [

      ],
      "spaceLength": 10,
      "stack": true,
      "steppedLine": false,
      "targets": [
        {
        "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\",job=\"$namespace/compactor\"}[5m])) by (level)",
        "interval": "",
        "legendFormat": "",
        "refId": "A"
        }
      ],
      "thresholds": [

      ],
      "timeFrom": null,
      "timeRegions": [

      ],
      "timeShift": null,
      "title": "Blocks Compacted",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": [

        ]
      },
      "yaxes": [
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        },
        {
        "format": "short",
        "label": null,
        "logBase": 1,
        "max": null,
        "min": null,
        "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
      }
    ],
    "refresh": "",
    "schemaVersion": 27,
    "style": "dark",
    "tags": [

    ],
    "templating": {
      "list": [
      {
        "current": {
        "selected": false,
        "text": "1-ops-tools1",
        "value": "1-ops-tools1"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "ds",
        "options": [

        ],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {
        "selected": false,
        "text": "loki-dev",
        "value": "loki-dev"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "logsds",
        "options": [

        ],
        "query": "loki",
        "queryValue": "",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {
        "selected": false,
        "text": "ops-tools1",
        "value": "ops-tools1"
        },
        "datasource": "$ds",
        "definition": "label_values(tempo_build_info, cluster)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "cluster",
        "options": [

        ],
        "query": {
        "query": "label_values(tempo_build_info, cluster)",
        "refId": "1-ops-tools1-cluster-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [

        ],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
        "selected": false,
        "text": "tracing-ops",
        "value": "tracing-ops"
        },
        "datasource": "$ds",
        "definition": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "namespace",
        "options": [

        ],
        "query": {
        "query": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)",
        "refId": "1-ops-tools1-namespace-Variable-Query"
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [

        ],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": ".*",
        "current": {
        "selected": false,
        "text": "All",
        "value": "$__all"
        },
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": true,
        "label": null,
        "multi": false,
        "name": "component",
        "options": [
        {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        {
          "selected": false,
          "text": "compactor",
          "value": "compactor"
        },
        {
          "selected": false,
          "text": "distributor",
          "value": "distributor"
        },
        {
          "selected": false,
          "text": "ingester",
          "value": "ingester"
        },
        {
          "selected": false,
          "text": "querier",
          "value": "querier"
        },
        {
          "selected": false,
          "text": "cortex-gw",
          "value": "cortex-gw"
        }
        ],
        "query": "compactor,distributor,ingester,querier,cortex-gw",
        "queryValue": "",
        "skipUrlSync": false,
        "type": "custom"
      }
      ]
    },
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {
      "refresh_intervals": [
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
      ]
    },
    "timezone": "",
    "title": "Tempo - Operational",
    "uid": "tempo-operational",
    "version": 1
    }
