Cortex Performance
Visualize Cortex Performance
Cortex is a High Available, Horizontally Scalable Prometheus as a Service
The use of this dashboard may vary depending on how you deploy Cortex. This assumes that all cortex pods have the label 'app' with the name of the cortex service associated.
Visualize Cortex Performance
{
  "__inputs": [
    {
      "name": "DS_TENANT-INTERNAL",
      "label": "prometheus",
      "description": "prometheus deployment scraping cortex services",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__requires": [
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "5.3.4"
    },
    {
      "type": "panel",
      "id": "graph",
      "name": "Graph",
      "version": "5.0.0"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "5.0.0"
    },
    {
      "type": "panel",
      "id": "singlestat",
      "name": "Singlestat",
      "version": "5.0.0"
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Visualize Cortex Performance\nCortex is a High Available, Horizontally Scalable Prometheus as a Service",
  "editable": true,
  "gnetId": 9820,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 80,
      "panels": [],
      "title": "Overview",
      "type": "row"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "description": "Number of metrics ingested per second. Note that a metric is essentially one data point in a metric series.",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 1
      },
      "id": 88,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": true
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "sum(rate(cortex_distributor_received_samples_total[1m]))",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
          "refId": "A"
        }
      ],
      "thresholds": "",
      "timeFrom": "1h",
      "title": "Metrics Ingested Per Second",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#d44a3a",
        "rgba(237, 129, 40, 0.89)",
        "#299c46"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "format": "percentunit",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 1
      },
      "id": 112,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": true
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "1 - (\nsum(rate(cortex_cassandra_request_duration_seconds_count{operation=\"SELECT\",status_code=~\"4.*|5.*\"}[5m]))\n/\nsum(rate(cortex_cassandra_request_duration_seconds_count{operation=\"SELECT\"}[5m]))\n)",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "0.5,0.75",
      "timeFrom": "1h",
      "title": "Cassandra READ Success Rate",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "description": "DynamoDB Write Capacity Units. AWS charges ~$0.000725 per WCU per hour (US-WEST-1 as of 1/03/19)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 9,
        "y": 1
      },
      "id": 96,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": true
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "sum(cortex_dynamo_table_capacity_units{op=\"write\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "",
      "timeFrom": "1h",
      "title": "Total Provisioned Write Capacity",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "description": "95 Percentile age of chunks stored in ingesters. Ideally we want chunks to last at least 12 hours so that queries on recent data is fast",
      "format": "dtdurations",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 12,
        "y": 1
      },
      "id": 90,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(cortex_ingester_chunk_age_seconds_bucket{app=\"ingester\"}) by (le))",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "",
      "timeFrom": "1h",
      "title": "Chunk Age (95%)",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 4
      },
      "id": 86,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": false
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "sum(rate(cortex_request_duration_seconds_count{route=\"api_prom_api_v1\"}[2m]))",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "",
      "timeFrom": "1h",
      "title": "Queries Per Second",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": true,
      "colors": [
        "#d44a3a",
        "rgba(237, 129, 40, 0.89)",
        "#299c46"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "format": "percentunit",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 6,
        "y": 4
      },
      "id": 109,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": true
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "1 - (\nsum(rate(cortex_cassandra_request_duration_seconds_count{operation=\"INSERT\",status_code=~\"4.*|5.*\"}[5m]))\n/\nsum(rate(cortex_cassandra_request_duration_seconds_count{operation=\"INSERT\"}[5m]))\n)",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "0.5,0.75",
      "timeFrom": "1h",
      "title": "Cassandra WRITE Success Rate",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "cacheTimeout": null,
      "colorBackground": false,
      "colorValue": false,
      "colors": [
        "#299c46",
        "rgba(237, 129, 40, 0.89)",
        "#d44a3a"
      ],
      "datasource": "${DS_TENANT-INTERNAL}",
      "description": "DynamoDB Read Capacity Units. AWS charges ~$0.000125 per WCU per hour (US-WEST-1 as of 1/03/19)",
      "format": "none",
      "gauge": {
        "maxValue": 100,
        "minValue": 0,
        "show": false,
        "thresholdLabels": false,
        "thresholdMarkers": true
      },
      "gridPos": {
        "h": 3,
        "w": 3,
        "x": 9,
        "y": 4
      },
      "id": 98,
      "interval": null,
      "links": [],
      "mappingType": 1,
      "mappingTypes": [
        {
          "name": "value to text",
          "value": 1
        },
        {
          "name": "range to text",
          "value": 2
        }
      ],
      "maxDataPoints": 100,
      "nullPointMode": "connected",
      "nullText": null,
      "postfix": "",
      "postfixFontSize": "50%",
      "prefix": "",
      "prefixFontSize": "50%",
      "rangeMaps": [
        {
          "from": "null",
          "text": "N/A",
          "to": "null"
        }
      ],
      "sparkline": {
        "fillColor": "rgba(31, 118, 189, 0.18)",
        "full": false,
        "lineColor": "rgb(31, 120, 193)",
        "show": true
      },
      "tableColumn": "",
      "targets": [
        {
          "expr": "sum(cortex_dynamo_table_capacity_units{op=\"read\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "refId": "A"
        }
      ],
      "thresholds": "",
      "timeFrom": "1h",
      "title": "Total Provisioned Read Capacity",
      "type": "singlestat",
      "valueFontSize": "80%",
      "valueMaps": [
        {
          "op": "=",
          "text": "N/A",
          "value": "null"
        }
      ],
      "valueName": "avg"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 7
      },
      "id": 2,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Success rate of fulfilled queries (from query-frontend)",
          "fill": 2,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 8
          },
          "id": 111,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "Errors",
              "color": "#bf1b00"
            },
            {
              "alias": "Success",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=~\"api_prom.*\",kubernetes_name=\"query-frontend\",status_code=~\"4.*|5.*\"}[5m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Errors",
              "refId": "B"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=~\"api_prom.*\",kubernetes_name=\"query-frontend\",status_code=~\"2.*\"}[5m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Success",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Query Error Rate",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "How long it takes queries to be executed once they reach query-frontend (perceived duration)",
          "fill": 2,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 8
          },
          "id": 110,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "500",
              "color": "#bf1b00"
            },
            {
              "alias": "200",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum by (le)(irate(cortex_request_duration_seconds_bucket{route=~\"api_prom.*\",kubernetes_name=\"query-frontend\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.95, sum by (le)(irate(cortex_request_duration_seconds_bucket{route=~\"api_prom.*\",kubernetes_name=\"query-frontend\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.50, sum by (le)(irate(cortex_request_duration_seconds_bucket{route=~\"api_prom.*\",kubernetes_name=\"query-frontend\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Query request duration",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "dtdurations",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "2xx": "#7EB26D",
            "4xx": "#F2C96D",
            "5xx": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 13
          },
          "id": 3,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{status_code=~\"2.*\", route=\"api_prom_api_v1\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "2xx",
              "refId": "A"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{status_code=~\"4.*\", route=\"api_prom_api_v1\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "4xx",
              "refId": "B"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{status_code=~\"5.*\", route=\"api_prom_api_v1\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "5xx",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Querier Read QPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorPrefix": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Queries in the queue for query-frontend. If this reaches 100, new query requests will be dropped with a HTTP 429 response",
          "format": "short",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": true,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 3,
            "w": 4,
            "x": 12,
            "y": 13
          },
          "id": 60,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "sum(cortex_query_frontend_queue_length{app=\"query-frontend\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": "20,80",
          "title": "Queued Queries",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "current"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "The average age of queries in the query-frontend queue",
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 8,
            "x": 16,
            "y": 13
          },
          "id": 58,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {}
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_query_frontend_queue_duration_seconds_sum[5m])) by (app)",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "{{ app }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Queued Queries Age",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Time queries sit in the queue before being executed\n95% of queries begin execution within this time period",
          "format": "dtdurations",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 3,
            "w": 4,
            "x": 12,
            "y": 16
          },
          "id": 84,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": false
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(cortex_query_frontend_queue_duration_seconds_bucket) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": "",
          "timeFrom": "1h",
          "timeShift": null,
          "title": "Query Queue Age (95%)",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "avg"
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 9,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 19
          },
          "id": 4,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_api_v1\"}[5m])) by (le))",
              "format": "time_series",
              "hide": false,
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_api_v1\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_api_v1\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Querier Read Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 9,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 19
          },
          "id": 56,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_query_frontend_queue_duration_seconds_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_query_frontend_queue_duration_seconds_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_query_frontend_queue_duration_seconds_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Query Age",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "dtdurations",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 9,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 24
          },
          "id": 48,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_chunk_store_chunks_per_query_bucket[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Chunks per Query",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "relevant for seriesstore",
          "fill": 9,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 24
          },
          "id": 46,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_chunk_store_index_lookups_per_query_bucket{app=\"querier\"}[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_chunk_store_index_lookups_per_query_bucket{app=\"querier\"}[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "D"
            },
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_chunk_store_index_lookups_per_query_bucket{app=\"querier\"}[5m])) by (le))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "E"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Index Lookups per Query",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 29
          },
          "id": 50,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(irate(cortex_chunk_store_chunks_per_query_sum{app=\"querier\"}[5m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Chunks",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Chunks per Query",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "relevant for seriesstore",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 29
          },
          "id": 52,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "avg(irate(cortex_chunk_store_index_lookups_per_query_sum{app=\"querier\"}[5m]))\n",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "Index Lookups",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Index Lookups per Query",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Read (Query)",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 8
      },
      "id": 5,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 12,
            "x": 0,
            "y": 65
          },
          "id": 19,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "TOTAL",
              "yaxis": 2
            },
            {
              "alias": "TOTAL",
              "color": "rgb(220, 220, 220)"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "irate(cortex_distributor_received_samples_total[5m])",
              "format": "time_series",
              "interval": "2m",
              "intervalFactor": 1,
              "legendFormat": "Tenant: {{ user }}",
              "refId": "A"
            },
            {
              "expr": "sum(irate(cortex_distributor_received_samples_total[5m]))",
              "format": "time_series",
              "interval": "2m",
              "intervalFactor": 1,
              "legendFormat": "TOTAL",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Cortex Ingestion Rate (metrics/s)",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Number of samples in ingesters to be flushed",
          "fill": 1,
          "gridPos": {
            "h": 7,
            "w": 12,
            "x": 12,
            "y": 65
          },
          "id": 82,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "/ingester.*/",
              "fill": 0,
              "yaxis": 2
            },
            {
              "alias": "All Ingesters",
              "color": "rgb(218, 218, 218)",
              "fill": 3
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(avg_over_time(cortex_ingester_flush_queue_length{app=\"ingester\"}[2m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "All Ingesters",
              "refId": "A"
            },
            {
              "expr": "avg_over_time(cortex_ingester_flush_queue_length{app=\"ingester\"}[2m])",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ instance }}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Ingester Flush Queue Length",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 72
          },
          "id": 25,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"2.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "2xx",
              "refId": "A"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"4.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "4xx",
              "refId": "B"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"5.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "5xx",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Distributor WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "failure": "#BF1B00",
            "success": "#7EB26D"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 72
          },
          "id": 6,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{status_code=\"success\", route=~\".*Query|.*LabelValues|.*MetricsForLabelMatchers\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "success",
              "refId": "A"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{status_code!=\"success\", route=~\".*Query|.*LabelValues|.*MetricsForLabelMatchers\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "failure",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Ingester QPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "failure": "#BF1B00",
            "success": "#7EB26D"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 76
          },
          "id": 29,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"2.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "2xx",
              "refId": "A"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"4.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "4xx",
              "refId": "B"
            },
            {
              "expr": "sum(irate(cortex_request_duration_seconds_count{route=\"api_prom_push\", status_code=~\"5.*\"}[5m]))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "5xx",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Ingester WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 78
          },
          "id": 7,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_request_duration_seconds_bucket{route=~\".*Query|.*LabelValues|.*MetricsForLabelMatchers\"}[5m])) by (le, route))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50 - {{ route }}",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_request_duration_seconds_bucket{route=~\".*Query|.*LabelValues|.*MetricsForLabelMatchers\"}[5m])) by (le, route))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99 - {{route}}",
              "refId": "B"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Read Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "2xx": "#7EB26D",
            "4xx": "#F2C96D",
            "5xx": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 80
          },
          "id": 27,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_push\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_push\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_request_duration_seconds_bucket{route=\"api_prom_push\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Distributor Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Write (Ingestion)",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 8,
      "panels": [
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "",
          "format": "none",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 3,
            "w": 4,
            "x": 8,
            "y": 66
          },
          "id": 92,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "sum(rate(cortex_cache_request_duration_seconds_count{method=\"memcache.store\"}[2m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": "",
          "timeFrom": "1h",
          "title": "Memcache WPS",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "avg"
        },
        {
          "cacheTimeout": null,
          "colorBackground": false,
          "colorValue": false,
          "colors": [
            "#299c46",
            "rgba(237, 129, 40, 0.89)",
            "#d44a3a"
          ],
          "datasource": "${DS_TENANT-INTERNAL}",
          "format": "none",
          "gauge": {
            "maxValue": 100,
            "minValue": 0,
            "show": false,
            "thresholdLabels": false,
            "thresholdMarkers": true
          },
          "gridPos": {
            "h": 3,
            "w": 4,
            "x": 12,
            "y": 66
          },
          "id": 94,
          "interval": null,
          "links": [],
          "mappingType": 1,
          "mappingTypes": [
            {
              "name": "value to text",
              "value": 1
            },
            {
              "name": "range to text",
              "value": 2
            }
          ],
          "maxDataPoints": 100,
          "nullPointMode": "connected",
          "nullText": null,
          "postfix": "",
          "postfixFontSize": "50%",
          "prefix": "",
          "prefixFontSize": "50%",
          "rangeMaps": [
            {
              "from": "null",
              "text": "N/A",
              "to": "null"
            }
          ],
          "sparkline": {
            "fillColor": "rgba(31, 118, 189, 0.18)",
            "full": false,
            "lineColor": "rgb(31, 120, 193)",
            "show": true
          },
          "tableColumn": "",
          "targets": [
            {
              "expr": "sum(rate(cortex_cache_request_duration_seconds_count{method=\"memcache-index.fetch\"}[2m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "refId": "A"
            }
          ],
          "thresholds": "",
          "timeFrom": "1h",
          "title": "Memcache QPS",
          "type": "singlestat",
          "valueFontSize": "80%",
          "valueMaps": [
            {
              "op": "=",
              "text": "N/A",
              "value": "null"
            }
          ],
          "valueName": "avg"
        },
        {
          "aliasColors": {
            "2xx": "#7EB26D",
            "5xx": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 0,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 69
          },
          "id": 33,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": true,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 3,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [
            {
              "alias": "/Total.*/i",
              "yaxis": 2
            },
            {
              "alias": "/Total.200.*/i",
              "color": "#629e51"
            },
            {
              "alias": "/Total.500.*/i",
              "color": "#bf1b00"
            },
            {
              "alias": "/.*- 200.*/",
              "dashes": true,
              "linewidth": 1,
              "spaceLength": 3
            }
          ],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "1 - (\nsum(irate(cortex_memcache_request_duration_seconds_count{method=~\"Memcache.Put.*\", status_code=~\"5.*\"}[5m]))\n/\nsum(irate(cortex_memcache_request_duration_seconds_count{method=~\"Memcache.Put.*\"}[5m]))\n)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "Success Rate",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Memcache Write Success Rate [2m]",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 2,
              "format": "percentunit",
              "label": null,
              "logBase": 1,
              "max": "1",
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "2xx": "#7EB26D",
            "5xx": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 0,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 69
          },
          "id": 9,
          "legend": {
            "alignAsTable": true,
            "avg": true,
            "current": true,
            "max": false,
            "min": true,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": true
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [
            {
              "alias": "/5xx/i",
              "yaxis": 2
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "1 - (\nsum(irate(cortex_memcache_request_duration_seconds_count{method=~\"Memcache.Get.*\", status_code=~\"5.*\"}[5m]))\n/\nsum(irate(cortex_memcache_request_duration_seconds_count{method=~\"Memcache.Get.*\"}[5m]))\n)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "2xx",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Memcache Query Success Rate [2m]",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "decimals": 2,
              "format": "percentunit",
              "label": null,
              "logBase": 1,
              "max": "1",
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 73
          },
          "id": 35,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.50, sum(rate(cortex_cache_request_duration_seconds_bucket{method=\"memcache.store\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_request_duration_seconds_bucket{method=\"memcache.store\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket{method=\"memcache.store\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Memcache Store Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 73
          },
          "id": 10,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_cache_request_duration_seconds_bucket{method=\"memcache-index.fetch\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_cache_request_duration_seconds_bucket{method=\"memcache-index.fetch\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_cache_request_duration_seconds_bucket{method=\"memcache-index.fetch\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Memcache Fetch Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 0,
            "y": 77
          },
          "id": 70,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "/.*500/i",
              "color": "#bf1b00"
            },
            {
              "alias": "/.*200/i",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_memcache_request_duration_seconds_count{method=\"Memcache.Put\",kubernetes_name=\"ingester\"}[5m])) by (kubernetes_name, status_code)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ status_code }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Ingester Memcached WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 6,
            "x": 6,
            "y": 77
          },
          "id": 72,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "/.*500/i",
              "color": "#bf1b00"
            },
            {
              "alias": "/.*200/i",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_memcache_request_duration_seconds_count{method=\"Memcache.Put\",kubernetes_name=\"querier\"}[5m])) by (kubernetes_name, status_code)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ status_code }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Querier Memcached WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 77
          },
          "id": 62,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "(sum(cortex_cache_hits) by (name, app))/(sum(cortex_cache_fetched_keys) by (name, app))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{app}} - {{ name }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Cache Hit Rate",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": false,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "percentunit",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 24,
            "x": 0,
            "y": 82
          },
          "id": 74,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum(rate(cortex_cache_request_duration_seconds_bucket{method=~\"memcache.*\"}[5m]))by (le))",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(cortex_cache_request_duration_seconds_bucket{method=~\"memcache.*\"}[5m]))by (le))",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "C"
            },
            {
              "expr": "histogram_quantile(0.5, sum(rate(cortex_cache_request_duration_seconds_bucket{method=~\"memcache.*\"}[5m]))by (le))",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "D"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Time Spent on memcache",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "PUT: Cache write requests\n\nNew PUT: Cache write requests with new entries\n\nEviction: Cache entries evicted",
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 0,
            "y": 86
          },
          "id": 76,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "PUT",
              "color": "#629e51"
            },
            {
              "alias": "New PUT",
              "color": "#1f78c1"
            },
            {
              "alias": "Eviction",
              "color": "#eab839"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(querier_cache_added_total[5m])) by (cache)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "PUT",
              "refId": "A"
            },
            {
              "expr": "sum(irate(querier_cache_added_new_total[5m])) by (cache)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "New PUT",
              "refId": "B"
            },
            {
              "expr": "sum(irate(querier_cache_evicted_total[5m])) by (cache)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "Eviction",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "FIFO Cache Writes",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Get: Read from cache request\n\nGet Stale: Get for an entry which expired\n\nGet Misses: Get for no valid entry",
          "fill": 1,
          "gridPos": {
            "h": 6,
            "w": 12,
            "x": 12,
            "y": 86
          },
          "id": 78,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "GET (stale)",
              "color": "#eab839"
            },
            {
              "alias": "GET (misses)",
              "color": "#bf1b00"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(querier_cache_gets_total[5m])) by (cache)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "GET",
              "refId": "A"
            },
            {
              "expr": "sum(irate(querier_cache_stale_gets_total[5m])) by (cache)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "GET (stale)",
              "refId": "B"
            },
            {
              "expr": "sum(irate(querier_cache_misses_total[5m])) by (cache)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "GET (misses)",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "FIFO Cache Reads",
          "tooltip": {
            "shared": true,
            "sort": 2,
            "value_type": "individual"
          },
          "transparent": false,
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Caching (Memcached)",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 11,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "DynamoDB Configured Write Capacity limits for each table",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 0,
            "y": 67
          },
          "id": 43,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(cortex_dynamo_table_capacity_units{op=\"write\"}) by (table)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ table }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "DynamoDB Write Quotas",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "DynamoDB Configured Read Capacity limits for each table",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 12,
            "x": 12,
            "y": 67
          },
          "id": 44,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(cortex_dynamo_table_capacity_units{op=\"read\"}) by (table)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ table }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "DynamoDB Read Quotas",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "200": "#7EB26D",
            "500": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 72
          },
          "id": 37,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{operation=\"DynamoDB.BatchWriteItem\"}[2m])) by (status_code)",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{status_code}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {
            "200": "#7EB26D",
            "500": "#BF1B00"
          },
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 10,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 72
          },
          "id": 12,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 0,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": true,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{operation=\"DynamoDB.BatchGetItemPages\"}[2m])) by (status_code)",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 1,
              "legendFormat": "{{status_code}}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "QPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Rate of \"ProvisionedThroughputExceededException\" errors per second from AWS APIs",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 76
          },
          "id": 41,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(cortex_dynamo_failures_total{operation=~\".*Write.*\"}[2m])) by (table, error)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ table }} - {{ error }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Write Errors",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Rate of \"ProvisionedThroughputExceededException\" errors per second from AWS APIs",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 76
          },
          "id": 15,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(irate(cortex_dynamo_failures_total{operation=~\".*Query.*\"}[5m])) by (table, error)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ table }} - {{ error }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Read Errors",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 0,
            "y": 80
          },
          "id": 39,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{operation=~\".*Write.*\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(cortex_dynamo_request_duration_seconds_bucket{operation=~\".*Write.*\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{operation=~\".*Write.*\"}[2m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 80
          },
          "id": 13,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null as zero",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "repeat": null,
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.5, sum(irate(cortex_dynamo_request_duration_seconds_bucket{operation=\"DynamoDB.BatchGetItemPages\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p50",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.95, sum(irate(cortex_dynamo_request_duration_seconds_bucket{operation=\"DynamoDB.BatchGetItemPages\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p95",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.99, sum(irate(cortex_dynamo_request_duration_seconds_bucket{operation=\"DynamoDB.BatchGetItemPages\"}[5m])) by (le))",
              "format": "time_series",
              "interval": "1m",
              "intervalFactor": 2,
              "legendFormat": "p99",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "rate of how many times cortex has to re-attempt queries against dynamodb. High values here indicate lots of errors or throttling in the backend, and can result in the query queue increasing.",
          "fill": 1,
          "gridPos": {
            "h": 4,
            "w": 12,
            "x": 12,
            "y": 84
          },
          "id": 54,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum(rate(cortex_dynamo_query_retry_count_count{operation!~\".*Write.*\"}[1m])) by (operation)",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ operation }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "DynamoDB Query Retries",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "repeat": null,
      "title": "Storage (DynamoDB)",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 11
      },
      "id": 100,
      "panels": [
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "description": "Rate of requests Cortex makes to CassandraDB.",
          "fill": 2,
          "gridPos": {
            "h": 5,
            "w": 10,
            "x": 0,
            "y": 37
          },
          "id": 102,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 2,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "500",
              "color": "#bf1b00"
            },
            {
              "alias": "200",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum by (status_code)(irate(cortex_cassandra_request_duration_seconds_count{operation=\"SELECT\"}[5m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ status_code }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "QPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 2,
          "gridPos": {
            "h": 5,
            "w": 10,
            "x": 10,
            "y": 37
          },
          "id": 104,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "connected",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"SELECT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.95, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"SELECT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.5, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"SELECT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Query Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 1,
          "gridPos": {
            "h": 5,
            "w": 10,
            "x": 0,
            "y": 42
          },
          "id": 106,
          "legend": {
            "alignAsTable": false,
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "rightSide": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [
            {
              "alias": "500",
              "color": "#bf1b00"
            },
            {
              "alias": "200",
              "color": "#629e51"
            }
          ],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "sum by (status_code)(irate(cortex_cassandra_request_duration_seconds_count{operation=\"INSERT\"}[5m]))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "{{ status_code }}",
              "refId": "A"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "WPS",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        },
        {
          "aliasColors": {},
          "bars": false,
          "dashLength": 10,
          "dashes": false,
          "datasource": "${DS_TENANT-INTERNAL}",
          "fill": 2,
          "gridPos": {
            "h": 5,
            "w": 10,
            "x": 10,
            "y": 42
          },
          "id": 107,
          "legend": {
            "avg": false,
            "current": false,
            "max": false,
            "min": false,
            "show": true,
            "total": false,
            "values": false
          },
          "lines": true,
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
          "percentage": false,
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
          "seriesOverrides": [],
          "spaceLength": 10,
          "stack": false,
          "steppedLine": false,
          "targets": [
            {
              "expr": "histogram_quantile(0.99, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"INSERT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p99",
              "refId": "B"
            },
            {
              "expr": "histogram_quantile(0.95, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"INSERT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p95",
              "refId": "A"
            },
            {
              "expr": "histogram_quantile(0.5, sum by (le)(irate(cortex_cassandra_request_duration_seconds_bucket{operation=\"INSERT\"}[5m])))",
              "format": "time_series",
              "intervalFactor": 1,
              "legendFormat": "p50",
              "refId": "C"
            }
          ],
          "thresholds": [],
          "timeFrom": null,
          "timeShift": null,
          "title": "Write Latency",
          "tooltip": {
            "shared": true,
            "sort": 0,
            "value_type": "individual"
          },
          "type": "graph",
          "xaxis": {
            "buckets": null,
            "mode": "time",
            "name": null,
            "show": true,
            "values": []
          },
          "yaxes": [
            {
              "format": "s",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            },
            {
              "format": "short",
              "label": null,
              "logBase": 1,
              "max": null,
              "min": null,
              "show": true
            }
          ],
          "yaxis": {
            "align": false,
            "alignLevel": null
          }
        }
      ],
      "title": "Storage (CassandraDB)",
      "type": "row"
    }
  ],
  "refresh": false,
  "schemaVersion": 16,
  "style": "dark",
  "tags": [
    "cortex"
  ],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "browser",
  "title": "Cortex Performance",
  "uid": "EucFH4rmz",
  "version": 2
}