apiVersion: v1
kind: ConfigMap
metadata:
  name: tempo-reads
data:
  tempo-reads.json: |-
    {
    "annotations": {
      "list": [

      ]
    },
    "editable": true,
    "gnetId": null,
    "graphTooltip": 0,
    "hideControls": false,
    "links": [
      {
      "asDropdown": true,
      "icon": "external link",
      "includeVars": true,
      "keepTime": true,
      "tags": [
        "tempo"
      ],
      "targetBlank": false,
      "title": "Tempo Dashboards",
      "type": "dashboards"
      }
    ],
    "refresh": "10s",
    "rows": [
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 1,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw\", route=\"tempo_api_traces_traceid\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 2,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw\",route=\"tempo_api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw\",route=\"tempo_api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw\",route=\"tempo_api_traces_traceid\"}[$__interval])) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw\",route=\"tempo_api_traces_traceid\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Gateway",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 3,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(jaeger_rpc_http_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/querier\", endpoint=\"/api/traces/-traceID-\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 4,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(jaeger_query_latency_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"get_trace\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(jaeger_query_latency_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"get_trace\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(jaeger_query_latency_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"get_trace\"}[$__interval])) * 1e3 / sum(rate(jaeger_query_latency_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"get_trace\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Jaeger Query",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 5,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=\"api_traces_traceid\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 6,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=\"api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=\"api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=\"api_traces_traceid\"}[$__interval])) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=\"api_traces_traceid\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Query Frontend",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 7,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=\"api_traces_traceid\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 8,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=\"api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=\"api_traces_traceid\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=\"api_traces_traceid\"}[$__interval])) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=\"api_traces_traceid\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Querier",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 9,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 10,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=\"/tempopb.Querier/FindTraceByID\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=\"/tempopb.Querier/FindTraceByID\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=\"/tempopb.Querier/FindTraceByID\"}[$__interval])) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/ingester\",route=\"/tempopb.Querier/FindTraceByID\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Ingester",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 11,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(cortex_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 12,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(cortex_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval])) * 1e3 / sum(rate(cortex_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "Memcached",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {
          "1xx": "#EAB839",
          "2xx": "#7EB26D",
          "3xx": "#6ED0E0",
          "4xx": "#EF843C",
          "5xx": "#E24D42",
          "error": "#E24D42",
          "success": "#7EB26D"
        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 10,
        "id": 13,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 0,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": true,
        "steppedLine": false,
        "targets": [
          {
          "expr": "sum by (status) (label_replace(label_replace(rate(tempodb_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\",   \"status_code\", \"([a-z]+)\"))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "{{status}}",
          "refId": "A",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "QPS",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 14,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 6,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(0.99, sum(rate(tempodb_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "99th Percentile",
          "refId": "A",
          "step": 10
          },
          {
          "expr": "histogram_quantile(0.50, sum(rate(tempodb_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) by (le)) * 1e3",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "50th Percentile",
          "refId": "B",
          "step": 10
          },
          {
          "expr": "sum(rate(tempodb_gcs_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval])) * 1e3 / sum(rate(tempodb_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__interval]))",
          "format": "time_series",
          "interval": "1m",
          "intervalFactor": 2,
          "legendFormat": "Average",
          "refId": "C",
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Latency",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "ms",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "GCS",
      "titleSize": "h6"
      },
      {
      "collapse": false,
      "height": "250px",
      "panels": [
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 15,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 4,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(.99, sum(rate(tempo_query_reads_bucket[$__interval])) by (layer, le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{layer}}",
          "legendLink": null,
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "p99",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 16,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 4,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(.5, sum(rate(tempo_query_reads_bucket[$__interval])) by (layer, le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "{{layer}}",
          "legendLink": null,
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "p50",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        },
        {
        "aliasColors": {

        },
        "bars": false,
        "dashLength": 10,
        "dashes": false,
        "datasource": "$datasource",
        "fill": 1,
        "id": 17,
        "legend": {
          "avg": false,
          "current": false,
          "max": false,
          "min": false,
          "show": true,
          "total": false,
          "values": false
        },
        "lines": true,
        "linewidth": 1,
        "links": [

        ],
        "nullPointMode": "null as zero",
        "percentage": false,
        "pointradius": 5,
        "points": false,
        "renderer": "flot",
        "seriesOverrides": [

        ],
        "spaceLength": 10,
        "span": 4,
        "stack": false,
        "steppedLine": false,
        "targets": [
          {
          "expr": "histogram_quantile(.99, sum(rate(tempo_query_bytes_read_bucket[$__interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "0.99",
          "legendLink": null,
          "step": 10
          },
          {
          "expr": "histogram_quantile(.9, sum(rate(tempo_query_bytes_read_bucket[$__interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "0.9",
          "legendLink": null,
          "step": 10
          },
          {
          "expr": "histogram_quantile(.5, sum(rate(tempo_query_bytes_read_bucket[$__interval])) by (le))",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "0.5",
          "legendLink": null,
          "step": 10
          }
        ],
        "thresholds": [

        ],
        "timeFrom": null,
        "timeShift": null,
        "title": "Bytes Read",
        "tooltip": {
          "shared": true,
          "sort": 0,
          "value_type": "individual"
        },
        "type": "graph",
        "xaxis": {
          "buckets": null,
          "mode": "time",
          "name": null,
          "show": true,
          "values": [

          ]
        },
        "yaxes": [
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": 0,
          "show": true
          },
          {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": false
          }
        ]
        }
      ],
      "repeat": null,
      "repeatIteration": null,
      "repeatRowId": null,
      "showTitle": true,
      "title": "TempoDB Access",
      "titleSize": "h6"
      }
    ],
    "schemaVersion": 14,
    "style": "dark",
    "tags": [
      "tempo"
    ],
    "templating": {
      "list": [
      {
        "current": {
        "text": "default",
        "value": "default"
        },
        "hide": 0,
        "label": null,
        "name": "datasource",
        "options": [

        ],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {
        "selected": true,
        "text": "All",
        "value": "$__all"
        },
        "datasource": "$datasource",
        "hide": 0,
        "includeAll": true,
        "label": "cluster",
        "multi": true,
        "name": "cluster",
        "options": [

        ],
        "query": "label_values(tempo_build_info, cluster)",
        "refresh": 1,
        "regex": "",
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [

        ],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": {
        "selected": true,
        "text": "All",
        "value": "$__all"
        },
        "datasource": "$datasource",
        "hide": 0,
        "includeAll": true,
        "label": "namespace",
        "multi": true,
        "name": "namespace",
        "options": [

        ],
        "query": "label_values(tempo_build_info, namespace)",
        "refresh": 1,
        "regex": "",
        "sort": 2,
        "tagValuesQuery": "",
        "tags": [

        ],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
      ]
    },
    "time": {
      "from": "now-1h",
      "to": "now"
    },
    "timepicker": {
      "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
      ],
      "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
      ]
    },
    "timezone": "utc",
    "title": "Tempo - Reads",
    "uid": "tempo-reads",
    "version": 1
    }
