apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-operational
data:
  loki-operational.json: |-
    {
      "annotations": {
        "list": [
          {
            "builtIn": 1,
            "datasource": "-- Grafana --",
            "enable": true,
            "hide": true,
            "iconColor": "rgba(0, 211, 255, 1)",
            "name": "Annotations & Alerts",
            "type": "dashboard"
          }
        ]
      },
      "editable": true,
      "gnetId": null,
      "graphTooltip": 1,
      "id": 16,
      "iteration": 1608299028629,
      "links": [],
      "panels": [
        {
          "collapsed": false,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 0
          },
          "id": 23,
          "panels": [],
          "title": "Overview",
          "type": "row"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 5,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 1
          },
          "hiddenSeries": false,
          "id": 26,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=~\"loki.*\", container=\"loki\"}[1m])",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "CPU Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 5,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 6,
            "y": 1
          },
          "hiddenSeries": false,
          "id": 27,
          "legend": {
            "avg": false,
            "current": false,
            "hideEmpty": false,
            "hideZero": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "container_memory_usage_bytes{namespace=\"$namespace\", pod=~\"loki.*\", container=\"loki\"}",
              "instant": false,
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Memory Usage",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "bytes",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": true,
          "dashLength": 10,
          "dashes": false,
          "datasource": "loki",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 1
          },
          "hiddenSeries": false,
          "id": 31,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": false,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "{}",
              "color": "#C4162A"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate({namespace=\"$namespace\"} |~ `\"level\":\"error\"`[1m]))",
              "hide": false,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Error rate",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": false
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "datasource": "loki",
          "fieldConfig": {
            "defaults": {
              "custom": {}
            },
            "overrides": []
          },
          "gridPos": {
            "h": 16,
            "w": 12,
            "x": 12,
            "y": 6
          },
          "id": 29,
          "options": {
            "showLabels": false,
            "showTime": false,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "panels": [],
          "targets": [
            {
              "expr": "{namespace=\"$namespace\"} |~ `\"level\":\"error\"`",
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "timeFrom": null,
          "timeShift": null,
          "title": "Errorl ogs",
          "type": "logs"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 0,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 8
          },
          "hiddenSeries": false,
          "id": 33,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (route)",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{route}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Success Rate",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 6,
            "y": 8
          },
          "hiddenSeries": false,
          "id": 32,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "rate(loki_distributor_ingester_append_failures_total{namespace=\"$namespace\"}[5m])",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Append Failures",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 0,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 34,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"$namespace\"}[5m])) by (pod)",
              "intervalFactor": 1,
              "legendFormat": "{{pod}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Bytes Received/Second",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "prometheus",
          "fieldConfig": {
            "defaults": {
              "custom": {},
              "links": []
            },
            "overrides": []
          },
          "fill": 1,
          "fillGradient": 0,
          "gridPos": {
            "h": 7,
            "w": 6,
            "x": 6,
            "y": 15
          },
          "hiddenSeries": false,
          "id": 35,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": false,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
          "options": {
            "alertThreshold": true
          },
          "panels": [],
          "percentage": false,
          "pluginVersion": "7.3.4",
          "pointradius": 1,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"$namespace\"}[5m])) by (pod)",
              "intervalFactor": 1,
              "legendFormat": "{{pod}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeRegions": [],
          "timeShift": null,
          "title": "Lines Received/Second",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 22
          },
          "id": 17,
          "panels": [
            {
              "aliasColors": {
                "5xx": "red"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 0,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 6,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (status) (\nlabel_replace(\n  label_replace(\n        rate(loki_request_duration_seconds_count{route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)",
                  "legendFormat": "{{status}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Queries/Second",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {
                "5xx": "red"
              },
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 4,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 7,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum by (status) (\nlabel_replace(\n  label_replace(\n          rate(loki_request_duration_seconds_count{route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n   \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
                  "legendFormat": "{{status}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Pushes/Second",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 10,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 8,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 11,
              "legend": {
                "avg": false,
                "current": false,
                "hideEmpty": false,
                "hideZero": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{exported_namespace=\"$namespace\"}[$__interval] offset 1h)))",
                  "legendFormat": "{{name}}-{{level}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Bad Words",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 12,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 2,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{namespace=\"$namespace\"}[5m])) by (tenant))",
                  "legendFormat": "{{tenant}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Lines Per Tenant (top 10)",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 16,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 4,
              "legend": {
                "avg": false,
                "current": false,
                "hideEmpty": true,
                "hideZero": true,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024",
                  "legendFormat": "{{tenant}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "MBs Per Tenant (Top 10)",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 4,
                "x": 20,
                "y": 23
              },
              "hiddenSeries": false,
              "id": 24,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"$namespace\"}[10m]) > 0",
                  "hide": false,
                  "interval": "",
                  "legendFormat": "{{container}}-{{pod}}",
                  "refId": "B"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Container Restarts",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 10,
                "w": 12,
                "x": 0,
                "y": 28
              },
              "hiddenSeries": false,
              "id": 9,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket{route=~\"api_prom_push|loki_api_v1_push\"})) * 1e3",
                  "legendFormat": ".99",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.75, sum by (le) (loki_request_duration_seconds_bucket{route=~\"api_prom_push|loki_api_v1_push\"})) * 1e3",
                  "legendFormat": ".9",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket{route=~\"api_prom_push|loki_api_v1_push\"})) * 1e3",
                  "legendFormat": ".5",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Push Latency",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 12,
                "y": 28
              },
              "hiddenSeries": false,
              "id": 12,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket)) * 1e3",
                  "legendFormat": ".99",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.9, sum by (le) (loki_request_duration_seconds_bucket)) * 1e3",
                  "legendFormat": ".9",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket)) * 1e3",
                  "legendFormat": ".5",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Distributor Latency",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 0,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 18,
                "y": 28
              },
              "hiddenSeries": false,
              "id": 71,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (route)",
                  "interval": "",
                  "legendFormat": "{{route}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Distributor Success Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "percentunit",
                  "label": "",
                  "logBase": 1,
                  "max": "1",
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 12,
                "y": 33
              },
              "hiddenSeries": false,
              "id": 13,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket{route=\"/logproto.Pusher/Push\"})) * 1e3",
                  "legendFormat": ".99",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.9, sum by (le) (loki_request_duration_seconds_bucket{route=\"/logproto.Pusher/Push\"})) * 1e3",
                  "hide": false,
                  "legendFormat": ".9",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket{route=\"/logproto.Pusher/Push\"})) * 1e3",
                  "hide": false,
                  "legendFormat": ".5",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Ingester Latency Write",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 0,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 18,
                "y": 33
              },
              "hiddenSeries": false,
              "id": 72,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)",
                  "interval": "",
                  "legendFormat": "{{route}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Ingester Success Rate Write",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "percentunit",
                  "label": "",
                  "logBase": 1,
                  "max": "1",
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 10,
                "w": 12,
                "x": 0,
                "y": 38
              },
              "hiddenSeries": false,
              "id": 10,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "hideEmpty": true,
                "hideZero": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
                  "legendFormat": "{{route}}-.99",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.9, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
                  "legendFormat": "{{route}}-.9",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
                  "legendFormat": "{{route}}-.5",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Query Latency",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 12,
                "y": 38
              },
              "hiddenSeries": false,
              "id": 14,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"})) * 1e3",
                  "legendFormat": ".99-{{route}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.9, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"})) * 1e3",
                  "legendFormat": ".9-{{route}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"})) * 1e3",
                  "legendFormat": ".5-{{route}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Querier Latency",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 0,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 18,
                "y": 38
              },
              "hiddenSeries": false,
              "id": 73,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (route)",
                  "interval": "",
                  "legendFormat": "{{route}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Querier Success Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "percentunit",
                  "label": "",
                  "logBase": 1,
                  "max": "1",
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "description": "",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 12,
                "y": 43
              },
              "hiddenSeries": false,
              "id": 15,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(0.99, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3",
                  "legendFormat": ".99-{{route}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(0.9, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3",
                  "legendFormat": ".9-{{route}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(0.5, sum by (le,route) (loki_request_duration_seconds_bucket{route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"})) * 1e3",
                  "legendFormat": ".5-{{route}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Ingester Latency Read",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 0,
              "fillGradient": 0,
              "gridPos": {
                "h": 5,
                "w": 6,
                "x": 18,
                "y": 43
              },
              "hiddenSeries": false,
              "id": 74,
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{namespace=\"$namespace\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)",
                  "interval": "",
                  "legendFormat": "{{route}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Ingester Success Rate Read",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "decimals": null,
                  "format": "percentunit",
                  "label": "",
                  "logBase": 1,
                  "max": "1",
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Queries",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 23
          },
          "id": 110,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 24
              },
              "hiddenSeries": false,
              "id": 112,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{namespace=\"$namespace\"}[1m])))",
                  "interval": "",
                  "legendFormat": "{{ tenant }} - {{ reason }}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Discarded Lines",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "columns": [],
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {}
                },
                "overrides": []
              },
              "fontSize": "100%",
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 24
              },
              "id": 113,
              "pageSize": null,
              "panels": [],
              "showHeader": true,
              "sort": {
                "col": 3,
                "desc": true
              },
              "styles": [
                {
                  "alias": "Time",
                  "align": "auto",
                  "dateFormat": "YYYY-MM-DD HH:mm:ss",
                  "pattern": "Time",
                  "type": "hidden"
                },
                {
                  "alias": "",
                  "align": "auto",
                  "colorMode": null,
                  "colors": [
                    "rgba(245, 54, 54, 0.9)",
                    "rgba(237, 129, 40, 0.89)",
                    "rgba(50, 172, 45, 0.97)"
                  ],
                  "dateFormat": "YYYY-MM-DD HH:mm:ss",
                  "decimals": 2,
                  "mappingType": 1,
                  "pattern": "tenant",
                  "thresholds": [],
                  "type": "string",
                  "unit": "short"
                },
                {
                  "alias": "",
                  "align": "auto",
                  "colorMode": null,
                  "colors": [
                    "rgba(245, 54, 54, 0.9)",
                    "rgba(237, 129, 40, 0.89)",
                    "rgba(50, 172, 45, 0.97)"
                  ],
                  "dateFormat": "YYYY-MM-DD HH:mm:ss",
                  "decimals": 2,
                  "mappingType": 1,
                  "pattern": "reason",
                  "thresholds": [],
                  "type": "number",
                  "unit": "short"
                },
                {
                  "alias": "",
                  "align": "right",
                  "colorMode": null,
                  "colors": [
                    "rgba(245, 54, 54, 0.9)",
                    "rgba(237, 129, 40, 0.89)",
                    "rgba(50, 172, 45, 0.97)"
                  ],
                  "decimals": 2,
                  "pattern": "/.*/",
                  "thresholds": [],
                  "type": "number",
                  "unit": "short"
                }
              ],
              "targets": [
                {
                  "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{namespace=\"$namespace\"}[1m])[$__range:1m])))",
                  "format": "table",
                  "instant": true,
                  "interval": "",
                  "legendFormat": "{{ tenant }} - {{ reason }}",
                  "refId": "A"
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Discarded Lines Per Interval",
              "transform": "table",
              "type": "table-old"
            }
          ],
          "title": "Limits",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 24
          },
          "id": 104,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 25
              },
              "hiddenSeries": false,
              "id": 106,
              "legend": {
                "avg": false,
                "current": false,
                "hideEmpty": true,
                "hideZero": true,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams))",
                  "interval": "",
                  "legendFormat": "{{ tenant }}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Active Streams",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 25
              },
              "hiddenSeries": false,
              "id": 108,
              "legend": {
                "avg": false,
                "current": false,
                "hideEmpty": true,
                "hideZero": true,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total[1m]) > 0))",
                  "interval": "",
                  "legendFormat": "{{ tenant }}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Streams Created/Sec",
              "tooltip": {
                "shared": false,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Streams",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 25
          },
          "id": 94,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 26
              },
              "hiddenSeries": false,
              "id": 102,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [
                {
                  "alias": "De-Dupe Ratio",
                  "yaxis": 2
                }
              ],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_ingester_chunks_flushed_total[1m]))",
                  "interval": "",
                  "legendFormat": "Chunks",
                  "refId": "A"
                },
                {
                  "expr": "sum(increase(cortex_chunk_store_deduped_chunks_total[1m]))/sum(increase(loki_ingester_chunks_flushed_total[1m])) < 1",
                  "interval": "",
                  "legendFormat": "De-Dupe Ratio",
                  "refId": "B"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Chunks Flushed/Sec",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "cards": {
                "cardPadding": null,
                "cardRound": null
              },
              "color": {
                "cardColor": "#b4ff00",
                "colorScale": "sqrt",
                "colorScheme": "interpolateSpectral",
                "exponent": 0.5,
                "mode": "spectrum"
              },
              "dataFormat": "tsbuckets",
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {}
                },
                "overrides": []
              },
              "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 26
              },
              "heatmap": {},
              "hideZeroBuckets": false,
              "highlightCards": true,
              "id": 100,
              "legend": {
                "show": true
              },
              "panels": [],
              "reverseYBuckets": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket[1m])) by (le)",
                  "format": "heatmap",
                  "instant": false,
                  "interval": "",
                  "legendFormat": "{{ le }}",
                  "refId": "A"
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Chunk Size Bytes",
              "tooltip": {
                "show": true,
                "showHistogram": false
              },
              "type": "heatmap",
              "xAxis": {
                "show": true
              },
              "xBucketNumber": null,
              "xBucketSize": null,
              "yAxis": {
                "decimals": 0,
                "format": "bytes",
                "logBase": 1,
                "max": null,
                "min": null,
                "show": true,
                "splitFactor": null
              },
              "yBucketBound": "auto",
              "yBucketNumber": null,
              "yBucketSize": null
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 7,
              "fillGradient": 0,
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 0,
                "y": 34
              },
              "hiddenSeries": false,
              "id": 96,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": true,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_ingester_chunks_flushed_total{reason=\"full\"}[1m]))/sum(rate(loki_ingester_chunks_flushed_total[1m]))",
                  "interval": "",
                  "legendFormat": "full",
                  "refId": "B"
                },
                {
                  "expr": "sum(rate(loki_ingester_chunks_flushed_total{reason=\"synced\"}[1m]))/sum(rate(loki_ingester_chunks_flushed_total[1m]))",
                  "interval": "",
                  "legendFormat": "synced",
                  "refId": "D"
                },
                {
                  "expr": "sum(rate(loki_ingester_chunks_flushed_total{reason=\"idle\"}[1m]))/sum(rate(loki_ingester_chunks_flushed_total[1m]))",
                  "hide": false,
                  "interval": "",
                  "legendFormat": "idle",
                  "refId": "A"
                },
                {
                  "expr": "sum(rate(loki_ingester_chunks_flushed_total{reason=\"max_age\"}[1m]))/sum(rate(loki_ingester_chunks_flushed_total[1m]))",
                  "interval": "",
                  "legendFormat": "max_age",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Chunk Flush Reason",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "percentunit",
                  "label": null,
                  "logBase": 1,
                  "max": "1",
                  "min": "0",
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "cards": {
                "cardPadding": null,
                "cardRound": null
              },
              "color": {
                "cardColor": "#b4ff00",
                "colorScale": "sqrt",
                "colorScheme": "interpolateSpectral",
                "exponent": 0.5,
                "max": null,
                "min": null,
                "mode": "spectrum"
              },
              "dataFormat": "tsbuckets",
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {}
                },
                "overrides": []
              },
              "gridPos": {
                "h": 9,
                "w": 12,
                "x": 12,
                "y": 34
              },
              "heatmap": {},
              "hideZeroBuckets": true,
              "highlightCards": true,
              "id": 98,
              "legend": {
                "show": true
              },
              "panels": [],
              "reverseYBuckets": false,
              "targets": [
                {
                  "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket[1m]))",
                  "format": "heatmap",
                  "instant": false,
                  "interval": "",
                  "legendFormat": "{{ le }}",
                  "refId": "A"
                }
              ],
              "timeFrom": null,
              "timeShift": null,
              "title": "Chunk Utilization",
              "tooltip": {
                "show": true,
                "showHistogram": false
              },
              "type": "heatmap",
              "xAxis": {
                "show": true
              },
              "xBucketNumber": null,
              "xBucketSize": null,
              "yAxis": {
                "decimals": 0,
                "format": "percentunit",
                "logBase": 1,
                "max": null,
                "min": null,
                "show": true,
                "splitFactor": null
              },
              "yBucketBound": "auto",
              "yBucketNumber": null,
              "yBucketSize": null
            }
          ],
          "title": "Chunks",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 26
          },
          "id": 114,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 27
              },
              "hiddenSeries": false,
              "id": 115,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fieldConfig": {
                "defaults": {
                  "custom": {},
                  "links": []
                },
                "overrides": []
              },
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 35
              },
              "hiddenSeries": false,
              "id": 116,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "alertThreshold": true
              },
              "panels": [],
              "percentage": false,
              "pluginVersion": "7.3.4",
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "BoltDB Shipper",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 27
          },
          "id": 52,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 30
              },
              "hiddenSeries": false,
              "id": 53,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (method, name, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{method}}-{{name}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (method, name, le))",
                  "hide": false,
                  "legendFormat": ".9-{{method}}-{{name}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_memcache_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (method, name, le))",
                  "hide": false,
                  "legendFormat": ".5-{{method}}-{{name}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 38
              },
              "hiddenSeries": false,
              "id": 54,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_memcache_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, method, name)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{method}}-{{name}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Memcached",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 28
          },
          "id": 57,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 31
              },
              "hiddenSeries": false,
              "id": 55,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 39
              },
              "hiddenSeries": false,
              "id": 58,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_consul_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, status_code, method)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Consul",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 29
          },
          "id": 90,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 36
              },
              "id": 91,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_cassandra_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_cassandra_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_cassandra_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 44
              },
              "id": 92,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_cassandra_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Cassandra",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 30
          },
          "id": 43,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 0,
                "y": 9
              },
              "hiddenSeries": false,
              "id": 41,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".9",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "MutateRows Latency",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 6,
                "y": 9
              },
              "hiddenSeries": false,
              "id": 46,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
                  "interval": "",
                  "intervalFactor": 1,
                  "legendFormat": "99%",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
                  "interval": "",
                  "legendFormat": "90%",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))",
                  "interval": "",
                  "legendFormat": "50%",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "ReadRows Latency",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 12,
                "y": 9
              },
              "hiddenSeries": false,
              "id": 44,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
                  "interval": "",
                  "intervalFactor": 1,
                  "legendFormat": "99%",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
                  "interval": "",
                  "legendFormat": "90%",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))",
                  "interval": "",
                  "legendFormat": "50%",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "GetTable Latency",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 18,
                "y": 9
              },
              "hiddenSeries": false,
              "id": 45,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".9",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_bigtable_request_duration_seconds_bucket{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "ListTables Latency",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 0,
                "y": 16
              },
              "hiddenSeries": false,
              "id": 47,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_bigtable_request_duration_seconds_count{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "MutateRows Status",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 6,
                "y": 16
              },
              "hiddenSeries": false,
              "id": 50,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_bigtable_request_duration_seconds_count{namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "ReadRows Status",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 12,
                "y": 16
              },
              "hiddenSeries": false,
              "id": 48,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_bigtable_request_duration_seconds_count{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "GetTable Status",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 7,
                "w": 6,
                "x": 18,
                "y": 16
              },
              "hiddenSeries": false,
              "id": 49,
              "interval": "",
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": false,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_bigtable_request_duration_seconds_count{namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "ListTables Status",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Big Table",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 31
          },
          "id": 76,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": null,
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 6,
                "x": 0,
                "y": 9
              },
              "id": 82,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_dynamo_failures_total{namespace=\"$namespace\"}[5m]))",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Failure Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": null,
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 6,
                "x": 6,
                "y": 9
              },
              "id": 83,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{namespace=\"$namespace\"}[5m]))",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Consumed Capacity Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": null,
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 6,
                "x": 12,
                "y": 9
              },
              "id": 84,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_dynamo_throttled_total{namespace=\"$namespace\"}[5m]))",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Throttled Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": null,
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 6,
                "x": 18,
                "y": 9
              },
              "id": 85,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_dynamo_dropped_requests_total{namespace=\"$namespace\"}[5m]))",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Dropped Rate",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": null,
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 6,
                "x": 0,
                "y": 15
              },
              "id": 86,
              "legend": {
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 2,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{namespace=\"$namespace\"}[5m])))",
                  "legendFormat": ".99",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{namespace=\"$namespace\"}[5m])))",
                  "legendFormat": ".9",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{namespace=\"$namespace\"}[5m])))",
                  "legendFormat": ".5",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Query Pages",
              "tooltip": {
                "shared": true,
                "sort": 0,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 9,
                "x": 6,
                "y": 15
              },
              "id": 87,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 6,
                "w": 9,
                "x": 15,
                "y": 15
              },
              "id": 88,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "Dynamo",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 32
          },
          "id": 60,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 33
              },
              "hiddenSeries": false,
              "id": 61,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_gcs_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 41
              },
              "hiddenSeries": false,
              "id": 62,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_gcs_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "GCS",
          "type": "row"
        },
        {
          "collapsed": true,
          "datasource": null,
          "gridPos": {
            "h": 1,
            "w": 24,
            "x": 0,
            "y": 33
          },
          "id": 78,
          "panels": [
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 10
              },
              "id": 79,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "histogram_quantile(.99, sum(rate(cortex_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "intervalFactor": 1,
                  "legendFormat": ".99-{{operation}}",
                  "refId": "A"
                },
                {
                  "expr": "histogram_quantile(.9, sum(rate(cortex_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".9-{{operation}}",
                  "refId": "B"
                },
                {
                  "expr": "histogram_quantile(.5, sum(rate(cortex_s3_request_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (operation, le))",
                  "hide": false,
                  "legendFormat": ".5-{{operation}}",
                  "refId": "C"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Latency By Operation",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            },
            {
              "aliasColors": {},
              "bars": false,
              "dashLength": 10,
              "dashes": false,
              "datasource": "prometheus",
              "fill": 1,
              "fillGradient": 0,
              "gridPos": {
                "h": 8,
                "w": 24,
                "x": 0,
                "y": 18
              },
              "id": 80,
              "interval": "",
              "legend": {
                "alignAsTable": true,
                "avg": false,
                "current": false,
                "max": false,
                "min": false,
                "rightSide": true,
                "show": true,
                "total": false,
                "values": false
              },
              "lines": true,
              "linewidth": 1,
              "nullPointMode": "null",
              "options": {
                "dataLinks": []
              },
              "panels": [],
              "percentage": false,
              "pointradius": 1,
              "points": false,
              "renderer": "flot",
              "seriesOverrides": [],
              "spaceLength": 10,
              "stack": false,
              "steppedLine": false,
              "targets": [
                {
                  "expr": "sum(rate(cortex_s3_request_duration_seconds_count{namespace=\"$namespace\"}[5m])) by (status_code, operation)",
                  "intervalFactor": 1,
                  "legendFormat": "{{status_code}}-{{operation}}",
                  "refId": "A"
                }
              ],
              "thresholds": [],
              "timeFrom": null,
              "timeRegions": [],
              "timeShift": null,
              "title": "Status By Method",
              "tooltip": {
                "shared": true,
                "sort": 2,
                "value_type": "individual"
              },
              "type": "graph",
              "xaxis": {
                "buckets": null,
                "mode": "time",
                "name": null,
                "show": true,
                "values": []
              },
              "yaxes": [
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                },
                {
                  "format": "short",
                  "label": null,
                  "logBase": 1,
                  "max": null,
                  "min": null,
                  "show": true
                }
              ],
              "yaxis": {
                "align": false,
                "alignLevel": null
              }
            }
          ],
          "title": "S3",
          "type": "row"
        }
      ],
      "refresh": "1m",
      "schemaVersion": 26,
      "style": "dark",
      "tags": [
        "logging",
        "loki"
      ],
      "templating": {
        "list": [
          {
            "allValue": null,
            "current": {
              "selected": false,
              "text": "logging",
              "value": "logging"
            },
            "datasource": "prometheus",
            "definition": "",
            "error": null,
            "hide": 0,
            "includeAll": false,
            "label": "namespace",
            "multi": false,
            "name": "namespace",
            "options": [],
            "query": "label_values(kube_pod_container_info, namespace)",
            "refresh": 1,
            "regex": "",
            "skipUrlSync": false,
            "sort": 1,
            "tagValuesQuery": "",
            "tags": [],
            "tagsQuery": "",
            "type": "query",
            "useTags": false
          }
        ]
      },
      "time": {
        "from": "now-1h",
        "to": "now"
      },
      "timepicker": {
        "refresh_intervals": [
          "10s",
          "30s",
          "1m",
          "5m",
          "15m",
          "30m",
          "1h",
          "2h",
          "1d"
        ]
      },
      "timezone": "",
      "title": "Loki - Operational",
      "uid": "loki-operational",
      "version": 2
    }